Synchronise master with upstream

.ansible-lint

@@ -0,0 +1,12 @@
+---
+exclude_paths:
+  - .cache/  # implicit unless exclude_paths is defined in config
+  - .zuul.d/
+offline: true
+parseable: true
+profile: basic
+skip_list:
+  - package-latest
+  - role-name
+strict: true
+use_default_rules: true

.zuul.d/base.yaml

@@ -126,6 +126,8 @@
     run: tests/playbooks/run.yml
     post-run: tests/playbooks/post.yml
     attempts: 5
+    roles:
+      - zuul: openstack/ansible-collection-kolla
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*

doc/source/admin/image-building.rst

@@ -525,6 +525,98 @@ The template becomes now:
    RUN cp /additions/jenkins/jenkins.json /jenkins.json
    {% endblock %}
 
+Custom docker templates
+-----------------------
+
+In order to unify the process of managing OpenStack-related projects, Kolla
+provides a way of building images for external 'non-built-in' projects.
+
+If the template for a 'non-built-in' project meets Kolla template standards,
+an operator can provide a root directory with a template via the
+``--docker-dir`` CLI option (can be specified multiple times).
+
+All Kolla's jinja2 macros should be available the same as for built-in
+projects with some notes:
+
+- The ``configure_user`` macro. As the 'non-built-in' user is unknown to Kolla,
+  there are no default values for user ID and group ID to use.
+  To use this macro, an operator should specify "non-default" user details
+  with ``<custom_user_name>-user`` configuration section and include info
+  for ``uid`` and ``gid`` at least.
+
+Let's look into how an operator can build an image for an in-house project
+with Kolla using `openstack/releases <https://opendev.org/openstack/releases>`_
+project.
+
+First, create a ``Dockerfile.j2`` template for the project.
+
+.. path /home/kolla/custom-kolla-docker-templates/releaser/Dockerfile.j2
+.. code-block:: jinja
+
+   FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }}
+
+   {% block labels %}
+   LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+   {% endblock %}
+
+   {% block releaser_header %}{% endblock %}
+
+   {% import "macros.j2" as macros with context %}
+
+   {{ macros.configure_user(name='releaser') }}
+
+   RUN ln -s releaser-source/* /releaser \
+       && {{ macros.install_pip(['/releaser-source']  | customizable("pip_packages")) }} \
+       && mkdir -p /etc/releaser \
+       && chown -R releaser: /etc/releaser \
+       && chmod 750 /etc/sudoers.d \
+       && touch /usr/local/bin/kolla_releaser_extend_start \
+       && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_releaser_extend_start
+
+   {% block footer %}{% endblock %}
+
+Suggested directory structure:
+
+.. code-block:: console
+
+   custom-kolla-docker-templates
+   |__ releaser
+       |__ Dockerfile.j2
+
+Then, modify Kolla's configuration so the engine can download sources and
+configure users.
+
+.. path /etc/kolla/kolla-build.conf
+.. code-block:: ini
+
+   [releaser]
+   type = git
+   location = https://opendev.org/openstack/releases
+   reference = master
+
+   [releaser-user]
+   uid = 53001
+   gid = 53001
+
+Last pre-check before building a new image - ensure that the new template
+is visible for Kolla:
+
+.. code-block:: console
+
+   $ kolla-build --list-images --docker-dir custom-kolla-docker-templates "^releaser$"
+   1 : base
+   2 : releaser
+   3 : openstack-base
+
+And finally, build the ``releaser`` image, passing the ``--docker-dir``
+argument:
+
+.. code-block:: console
+
+   kolla-build --docker-dir custom-kolla-docker-templates "^releaser$"
+
+Can I use the ``--template-override`` option for custom templates? Yes!
+
 Custom repos
 ------------
 

docker/cinder/cinder-base/Dockerfile.j2

@@ -16,13 +16,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'ceph-common',
         'cryptsetup',
         'lvm2',
+        'nvme-cli',
         'qemu-img'
     ] %}
 {% elif base_package_type == 'deb' %}
     {% set cinder_base_packages = [
         'ceph-common',
-        'lvm2',
         'cryptsetup',
+        'lvm2',
+        'nvme-cli',
         'python3-cephfs',
         'python3-rados',
         'python3-rbd',

docker/cinder/cinder-volume/Dockerfile.j2

@@ -12,7 +12,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'lsscsi',
         'device-mapper-multipath',
         'nfs-utils',
-        'nvme-cli',
         'nvmetcli',
         'sysfsutils',
         'targetcli'
@@ -22,7 +21,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'lsscsi',
         'multipath-tools',
         'nfs-common',
-        'nvme-cli',
         'sysfsutils',
         'targetcli-fb',
         'thin-provisioning-tools',

docker/macros.j2

@@ -43,6 +43,9 @@
 {% endmacro %}
 
 {% macro configure_user(name, groups=None, shell=None, homedir=None) %}
+{%- if name not in users %}
+{{ raise_error("Failed to find configuration for '" + name + "' user. Try specifying '" + name + "-user' config section.") }}
+{%- endif %}
 {% set user=users[name] %}
 {%- if not homedir %}
 {% set homedir='/var/lib/' + name %}

docker/ovn/ovn-sb-db-relay/Dockerfile.j2

@@ -0,0 +1,15 @@
+FROM {{ namespace }}/{{ image_prefix }}ovn-sb-db-server:{{ tag }}
+{% block labels %}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+{% endblock %}
+
+{% block ovn_sb_db_server_header %}{% endblock %}
+
+{% block ovn_sb_db_relay_ovn_ctl %}
+{# TODO(mnasiadka): Switch to 25.03 branch when available, ideally rpm/deb packages will be
+                    available. #}
+RUN curl -o /usr/share/ovn/scripts/ovn-ctl https://raw.githubusercontent.com/ovn-org/ovn/refs/heads/main/utilities/ovn-ctl
+{% endblock %}
+
+{% block ovn_sb_db_relay_footer %}{% endblock %}
+{% block footer %}{% endblock %}

kolla/image/kolla_worker.py

@@ -271,6 +271,7 @@ def _get_methods(self):
         return {
             'debian_package_install': jinja_methods.debian_package_install,
             'handle_repos': jinja_methods.handle_repos,
+            'raise_error': jinja_methods.raise_error,
         }
 
     def get_users(self):
@@ -280,7 +281,17 @@ def get_users(self):
         for section in all_sections:
             match = re.search('^.*-user$', section)
             if match:
-                user = self.conf[match.group(0)]
+                cfg_group_name = match.group(0)
+
+                if cfg_group_name not in self.conf._groups:
+                    self.conf.register_opts(
+                        common_config.get_user_opts(
+                            None, None,
+                            # cut `-user` suffix
+                            group=cfg_group_name[:-5]),
+                        group=cfg_group_name
+                    )
+                user = self.conf[cfg_group_name]
                 ret[match.group(0)[:-5]] = {
                     'uid': user.uid,
                     'gid': user.gid,

kolla/image/tasks.py

@@ -232,6 +232,7 @@ def reset_userinfo(tarinfo):
                 git.Git().clone(source['source'], clone_dir)
                 git.Git(clone_dir).checkout(source['reference'])
                 reference_sha = git.Git(clone_dir).rev_parse('HEAD')
+                git.Git(clone_dir).remote("remove", "origin")
                 self.logger.debug("Git checkout by reference %s (%s)",
                                   source['reference'], reference_sha)
             except Exception as e:

kolla/template/methods.py

@@ -11,6 +11,8 @@
 # limitations under the License.
 
 import os
+import typing as t
+
 import yaml
 
 from jinja2 import pass_context
@@ -150,3 +152,7 @@ def handle_repos(context, reponames, mode):
         commands = "RUN %s" % commands
 
     return commands
+
+
+def raise_error(msg: str) -> t.NoReturn:
+    raise Exception(msg)

releasenotes/notes/bring-configure_user-macro-to-custom-templates-61c143326a35c7ed.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Extends the support of externally-managed projects provided by the
+    ``--docker-dir`` option with an ability to use ``configure_user``
+    jinja2 macros like Kolla built-in projects.
+    The operator should specify "non-default" user details with
+    ``<custom_user_name>-user`` configuration section and include info for
+    ``uid`` and ``gid`` at least.

releasenotes/notes/bug-2098904-4c5670049a7e1a66.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Removes Git remote URLs after cloning to prevent credential exposure.
+    `LP#2098904 <https://bugs.launchpad.net/kolla/+bug/2098904>`__

roles/configure-ephemeral/meta/main.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - ansible.posix
+  - community.general

roles/configure-ephemeral/tasks/main.yml

@@ -3,23 +3,23 @@
 # unallocated ephemeral device attached at /dev/xvde
 - name: Set ephemeral device if /dev/xvde exists
   when: ansible_devices["xvde"] is defined
-  set_fact:
+  ansible.builtin.set_fact:
     ephemeral_device: "/dev/xvde"
 
 # On other providers, we have a device called "ephemeral0".
 - name: Set ephemeral device by label
   when: ephemeral_device is undefined
   block:
     - name: Get ephemeral0 device node
-      command: /sbin/blkid -L ephemeral0
+      ansible.builtin.command: /sbin/blkid -L ephemeral0
       register: ephemeral0
       # rc !=0 is expected
-      failed_when: False
-      changed_when: False
+      failed_when: false
+      changed_when: false
 
     - name: Set ephemeral device if LABEL exists
       when: "ephemeral0.rc == 0"
-      set_fact:
+      ansible.builtin.set_fact:
         ephemeral_device: "{{ ephemeral0.stdout }}"
 
 - name: Configure additional disk (if available)
@@ -53,13 +53,13 @@
         fstype: ext4
         dev: "{{ ephemeral_device }}"
 
-    - name: "Ensure {{ configure_ephemeral_mountpoint }} mountpoint is created"
+    - name: "Ensure mountpoint {{ configure_ephemeral_mountpoint }}"
       ansible.builtin.file:
         path: "{{ configure_ephemeral_mountpoint }}"
         owner: root
         group: root
         state: directory
-        mode: 0755
+        mode: "0755"
 
     - name: Mount additional filesystem
       ansible.posix.mount:

roles/kolla-build-config/tasks/main.yml

@@ -1,5 +1,8 @@
 ---
 - name: Ensure kolla-build.conf exists
-  template:
+  ansible.builtin.template:
     src: kolla-build.conf.j2
     dest: "{{ kolla_build_config_path }}"
+    mode: "0660"
+    owner: zuul
+    group: zuul

roles/kolla-build-deps/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- name: Create dir for kolla logs
+  ansible.builtin.file:
+    path: "{{ kolla_logs_dir }}"
+    state: directory
+    mode: "0755"
+    owner: zuul
+    group: zuul
+
+- name: Dump host info to logs
+  ansible.builtin.command: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/tools/dump_info.sh"
+  args:
+    chdir: "{{ kolla_logs_dir }}"
+  changed_when: true
+
+- name: Create dir for kolla build logs
+  ansible.builtin.file:
+    path: "{{ kolla_build_logs_dir }}"
+    state: directory
+    mode: "0755"
+    owner: zuul
+    group: zuul
+
+- name: Install Python3 pip and setuptools
+  ansible.builtin.package:
+    name:
+      - python3-pip
+      - python3-setuptools
+  become: true
+
+# NOTE(hrw): On RedHat systems venv is part of python3-libs
+- name: Install Python3 venv on Debian systems
+  ansible.builtin.package:
+    name:
+      - python3-venv
+  become: true
+  when: ansible_facts.os_family == "Debian"
+
+- name: Install kolla in a venv
+  ansible.builtin.pip:
+    name:
+      - "pip"
+      - "."
+    state: latest
+    virtualenv: "{{ virtualenv_path }}"
+    virtualenv_command: "python3.{{ ansible_facts.python.version.minor }} -m venv"
+    chdir: "{{ zuul.project.src_dir }}"
+
+- name: Configure container engine
+  ansible.builtin.include_role:
+    name: "{{ container_engine }}"
+
+- name: Install podman/docker_sdk in a venv
+  vars:
+    virtualenv: "{{ virtualenv_path }}"
+    docker_sdk_virtualenv_owner: "{{ lookup('env', 'USER') }}"
+  ansible.builtin.include_role:
+    name: "{{ container_engine }}_sdk"
+
+- name: Ensure container engine socket is world-writable
+  ansible.builtin.file:
+    path: "{{ '/run/docker.sock' if container_engine == 'docker' else '/run/podman/podman.sock' }}"
+    mode: "0666"
+  become: true