[bop]Build RPM pkgs from OpenDev Depends On

ci/playbooks/tcib/tcib.yml

@@ -26,7 +26,6 @@
 
     - name: Build tcib package
       vars:
-        cifmw_bop_openstack_project_path: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/tcib"
         cifmw_bop_yum_repos_dir: "{{ cifmw_build_containers_repo_dir }}"
         cifmw_bop_gating_repo_dest: "{{ cifmw_build_containers_repo_dir }}"
       tags:

roles/build_openstack_packages/README.md

@@ -4,7 +4,7 @@ An Ansible role for generating custom RPMSs of OpenStack Projects using DLRN
 ## Parameters
 
 * `cifmw_bop_dlrn_deps`: (List) A list of DLRN project system dependencies.
-* `cifmw_bop_build_repo_dir`: (String) The directory where the DLRN repo is built. Defaults to `~/`.
+* `cifmw_bop_build_repo_dir`: (String) The directory where the DLRN repo is built. Defaults to `{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/logs`.
 * `cifmw_bop_dlrn_repo_url`: (String) The URL of the DLRN repository.
 * `cifmw_bop_dlrn_from_source`: (String) Install DLRN from git. Defaults to `False`.
 * `cifmw_bop_rdoinfo_repo_url`: (String) The URL of the rdoinfo repository that contains the project definitions for DLRN.
@@ -19,15 +19,59 @@ An Ansible role for generating custom RPMSs of OpenStack Projects using DLRN
 * `cifmw_bop_gating_repo_dest`: (String) The path of directory to store gating repo file and repo metadata.
   Defaults to `cifmw_bop_build_repo_dir` var.
 * `cifmw_bop_dlrn_cleanup`: (Boolean) Clean up the DLRN artifacts. Defaults to `False`.
-
-### TODO
-- Allow building multiple openstack projects
-- Build RDO dist-git changes
-- Parse gerrit review and clone the respective project and build it
-- Build rpm from Zuul Depends On
+* `cifmw_bop_branchless_projects`: List of project does not have stable branches.
+* `cifmw_bop_skipped_projects`: List of projects on which DLRN build needs to be skipped.
+* `cifmw_bop_change_list`: Zuul Change list constructed while using Depends-On in CI to build rpm packages using DLRN.
+* `cifmw_bop_release_mapping`: A list of openstack release names and their respective branch names.
 
 ## Examples
 
+### Construct cifmw_bop_change_list list
+
+cifmw_bop_change_list var contains a list of dict containing branch, change number,
+host, project name and change refspec. It is used by build_openstack_packages to
+download the spec change of a project and build rpm out of that.
+
+cifmw_bop_change_list is constructed from a OpenDev Depends-On based on
+following conditions:
+- Depends-On project is not in cifmw_bop_skipped_projects list.
+- Depends-On project is not a RDO project.
+- Zuul override-checkout branch should be similar to project branch.
+  (It is useful in case of downstream).
+- Depends-On project branch is in cifmw_bop_release_mapping.
+
+We want to build DLRN rpm packages from following reviews:
+* https://review.opendev.org/c/openstack/neutron/+/913763 - This code
+  review is proposed against stable/2023.1 branch. If we open this url, we can
+  get the refspec from the downstream the patch section.
+
+```
+cifmw_bop_change_list:
+  - "branch": "stable/2023.1"
+    "change": "913763"
+    "host": "https://review.opendev.org"
+    "project": "openstack/neutron"
+    "refspec": "refs/changes/63/913763/3"
+```
+* Extend the above cifmw_bop_change_list change to include
+  https://github.com/openstack-k8s-operators/tcib/pull/162 also.
+```
+cifmw_bop_change_list:
+  - "branch": "stable/2023.1"
+    "change": "913763"
+    "host": "https://review.opendev.org"
+    "project": "openstack/neutron"
+    "refspec": "refs/changes/63/913763/3"
+
+  - "branch": "main",
+    "change": "162",
+    "host": "https://github.com"
+    "project": "openstack-k8s-operators/tcib"
+    "refspec": "refs/changes/62/162/60ad76dbeacb491c0bddca419c61d398e83fe1af"
+```
+
+### Run the role
+
 ```
 - hosts: localhost
   vars:

roles/build_openstack_packages/defaults/main.yml

@@ -31,7 +31,7 @@ cifmw_bop_dlrn_deps:
   - sqlite
   - python3-libselinux
 
-cifmw_bop_build_repo_dir: "{{ ansible_user_dir }}"
+cifmw_bop_build_repo_dir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/logs"
 cifmw_bop_dlrn_repo_url: "https://github.com/openstack-packages/DLRN.git"
 cifmw_bop_dlrn_from_source: false
 
@@ -57,12 +57,50 @@ cifmw_bop_use_components: 1
 
 cifmw_bop_yum_repos_dir: "/etc/yum.repos.d"
 
-cifmw_bop_openstack_release: master
-# cifmw_bop_osp_release: 18.0
+cifmw_bop_openstack_release: antelope
+# cifmw_bop_osp_release: rhos-18.0
 cifmw_bop_openstack_project_path: ''
 
 cifmw_bop_gating_repo_dest: "{{ cifmw_bop_build_repo_dir }}"
 cifmw_bop_dlrn_cleanup: false
 
 cifmw_bop_timestamper_cmd: >-
   | awk '{ print strftime("%Y-%m-%d %H:%M:%S |"), $0; fflush(); }'
+
+cifmw_bop_branchless_projects:
+  - openstack-k8s-operators/tcib
+
+cifmw_bop_change_list: []
+
+cifmw_bop_release_mapping:
+  master: master
+  antelope: stable/2023.1
+
+cifmw_bop_skipped_projects:
+  - testproject
+  - openstack-k8s-operators/ci-framework
+  - opendev.org/zuul/zuul-jobs
+  - openstack-k8s-operators/barbican-operator
+  - openstack-k8s-operators/cinder-operator
+  - openstack-k8s-operators/dataplane-operator
+  - openstack-k8s-operators/designate-operator
+  - openstack-k8s-operators/glance-operator
+  - openstack-k8s-operators/heat-operator
+  - openstack-k8s-operators/horizon-operator
+  - openstack-k8s-operators/infra-operator
+  - openstack-k8s-operators/install_yamls
+  - openstack-k8s-operators/ironic-operator
+  - openstack-k8s-operators/keystone-operator
+  - openstack-k8s-operators/manila-operator
+  - openstack-k8s-operators/mariadb-operator
+  - openstack-k8s-operators/neutron-operator
+  - openstack-k8s-operators/nova-operator
+  - openstack-k8s-operators/octavia-operator
+  - openstack-k8s-operators/openstack-ansibleee-operator
+  - openstack-k8s-operators/openstack-baremetal-operator
+  - openstack-k8s-operators/openstack-operator
+  - openstack-k8s-operators/ovn-operator
+  - openstack-k8s-operators/placement-operator
+  - openstack-k8s-operators/repo-setup
+  - openstack-k8s-operators/swift-operator
+  - openstack-k8s-operators/telemetry-operator

roles/build_openstack_packages/molecule/default/converge.yml

@@ -20,8 +20,7 @@
   vars:
     ansible_user_dir: "{{ lookup('env', 'HOME') }}"
     cifmw_basedir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
-    cifmw_bop_dlrn_target: centos9-stream
-    cifmw_bop_initial_dlrn_config: centos9-local
+    cifmw_bop_openstack_release: master
     cifmw_bop_dlrn_baseurl: https://trunk.rdoproject.org/centos9-master
     cifmw_bop_yum_repos_dir: "{{ cifmw_basedir }}/artifacts/repositories/"
   pre_tasks:

roles/build_openstack_packages/tasks/main.yml

@@ -17,8 +17,8 @@
 - name: Install dlrn
   ansible.builtin.import_tasks: install_dlrn.yml
 
-- name: Run dlrn
-  ansible.builtin.import_tasks: run_dlrn.yml
+- name: Parse Zuul changes and Build RPM packages
+  ansible.builtin.import_tasks: parse_and_build_pkgs.yml
 
 - name: Create repo
   ansible.builtin.import_tasks: create_repo.yml

roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml

@@ -0,0 +1,52 @@
+---
+- name: Parse Zuul changes
+  with_items: "{{ zuul['items'] }}"
+  when:
+    - zuul is defined
+    - "'change_url' in item"
+    - '"-distgit" not in item.project'
+    - item.project.name not in cifmw_bop_change_list|default([]) | map(attribute='project') |list
+    - >-
+        cifmw_bop_release_mapping[cifmw_bop_openstack_release] in item.branch or
+        (cifmw_bop_openstack_release is not defined and item.branch == zuul.override_checkout | default(zuul.branch)) or
+        (cifmw_bop_osp_release is defined and cifmw_bop_osp_release in item.branch) or
+        item.project.name in cifmw_bop_branchless_projects
+  ansible.builtin.set_fact:
+    cacheable: true
+    cifmw_bop_change_list:
+      "{{ cifmw_bop_change_list|default([]) +
+          [{'host': item.change_url | regex_search('(^https?://.*?)/', '\\1') | first,
+            'project': item.project.name,
+            'branch': item.branch,
+            'change': item.change,
+            'refspec': '/'.join(['refs', 'changes',
+                                  item.change[-2:],
+                                  item.change,
+                                  item.patchset]) }] }}"
+
+
+- name: Print Zuul change list
+  ansible.builtin.debug:
+    var: cifmw_bop_change_list
+
+- name: Build DLRN packages from zuul changes
+  when:
+    - '"-distgit" not in _change.project'
+    - _change.project not in cifmw_bop_skipped_projects
+    - >-
+        cifmw_bop_release_mapping[cifmw_bop_openstack_release] in _change.branch or
+        (zuul.project.name | default("") == cifmw_bop_rdoinfo_repo_name) or
+        (cifmw_bop_osp_release is defined and cifmw_bop_osp_release in _change.branch) or
+        _change.project in cifmw_bop_branchless_projects
+  loop: "{{ cifmw_bop_change_list }}"
+  loop_control:
+    loop_var: "_change"
+  ansible.builtin.include_tasks: run_dlrn.yml
+
+- name: Build DLRN packages from Local change
+  when: cifmw_bop_openstack_project_path | length > 0
+  vars:
+    _change:
+      branch: "{{ cifmw_bop_openstack_release }}"
+      project: "{{ cifmw_bop_openstack_project_path | basename }}"
+  ansible.builtin.include_tasks: run_dlrn.yml

roles/build_openstack_packages/tasks/run_dlrn.yml

@@ -14,15 +14,25 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Set proper branch name for stable opendev branches
+  ansible.builtin.set_fact:
+    _project_branch: "{{ cifmw_bop_openstack_release }}"
+  when: _change.branch != cifmw_bop_openstack_release
+
+- name: Set proper branch name for non-stable branches
+  ansible.builtin.set_fact:
+    _project_branch: "{{ _change.branch }}"
+  when: _change.branch == cifmw_bop_openstack_release
+
 - name: Set RDO release for master openstack release
-  when: cifmw_bop_openstack_release in ['master']
+  when: _project_branch in ['master']
   ansible.builtin.set_fact:
-    cifmw_bop_rdo_release: "rpm-{{ cifmw_bop_openstack_release }}"
+    cifmw_bop_rdo_release: "rpm-{{ _project_branch }}"
 
 - name: Set RDO release for non-master openstack release
-  when: cifmw_bop_openstack_release not in ['master']
+  when: _project_branch not in ['master']
   ansible.builtin.set_fact:
-    cifmw_bop_rdo_release: "{{ cifmw_bop_openstack_release }}-rdo"
+    cifmw_bop_rdo_release: "{{ _project_branch }}-rdo"
 
 - name: Make sure /usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT exists (RHEL)
   ansible.builtin.shell:
@@ -35,16 +45,23 @@
   when: ansible_distribution in ['RedHat']
 
 - name: Override projects.ini settings
+  vars:
+    _source_branch: >-
+      {% if _change.project in cifmw_bop_branchless_projects -%}
+      'master'
+      {%- else -%}
+      {{ cifmw_bop_release_mapping[_project_branch] }}
+      {%- endif -%}
   ansible.builtin.lineinfile:
     dest: '{{ cifmw_bop_build_repo_dir }}/DLRN/projects.ini'
     regexp: '{{ item.regexp }}'
     line: '{{ item.line }}'
   with_items:
     - {regexp: 'baseurl=.*', line: 'baseurl={{ cifmw_bop_dlrn_baseurl }}'}
     - {regexp: 'distro=.*', line: 'distro={{ cifmw_bop_rdo_release }}'}
-    - {regexp: 'source=.*', line: 'source={{ cifmw_bop_openstack_release }}'}
+    - {regexp: 'source=.*', line: 'source={{ _source_branch }}'}
 
-- name: Mapp project
+- name: Map project
   block:
     - name: Map project name to DLRN project name
       register: project_name_mapped
@@ -55,7 +72,7 @@
           set -xeo pipefail
           # {{ cifmw_bop_rdoinfo_repo_name }}/{{ cifmw_bop_rdoinfo_repo_name.split('info')[0] }}-full.yml will
           # return rdo-full.yml and for downstream is osp-full.yml.
-          rdopkg findpkg -s '{{ cifmw_bop_openstack_project_path | basename }}' \
+          rdopkg findpkg -s '{{ _change.project | basename }}' \
             -i {{ cifmw_bop_rdoinfo_repo_name }}/{{ cifmw_bop_rdoinfo_repo_name.split('info')[0] }}-full.yml | \
             awk '/^name/{print $2}; {print "findpkg: " $0 > "/dev/stderr"}'
       changed_when: false
@@ -76,25 +93,59 @@
 - name: Mapping succeeded
   when: project_name_mapped is success
   block:
-    - name: Append project name to package list
-      ansible.builtin.set_fact:
-        artg_rdo_packages: "{{ project_name_mapped.stdout }}"
-
     - name: Create data directory if doesn't exist yet
       ansible.builtin.file:
         path: "{{ cifmw_bop_build_repo_dir }}/DLRN/data/"
         state: directory
 
-    - name: "Check for existing {{ project_name_mapped.stdout }}"
+    - name: "Check for existing {{ project_name_mapped.stdout }}" # noqa: name[template]
       ansible.builtin.stat:
         path: '{{ cifmw_bop_build_repo_dir }}/DLRN/data/{{ project_name_mapped.stdout }}/.git'
       register: repo_status
 
-    - name: "Clone {{ project_name_mapped.stdout }}"  # noqa: latest[git]
+    - name: "Sync {{ project_name_mapped.stdout }} from local repo" # noqa: name[template]
+      when:
+        - cifmw_bop_openstack_project_path | length > 0
+        - not repo_status.stat.exists
       ansible.builtin.git:
         accept_hostkey: true
         repo: '{{ cifmw_bop_openstack_project_path }}'
         dest: '{{ cifmw_bop_build_repo_dir }}/DLRN/data/{{ project_name_mapped.stdout }}'
+        version: '{{ _change.branch }}'
+
+    - name: "Clone {{ project_name_mapped.stdout }} from Github" # noqa: name[template]
+      when:
+        - cifmw_bop_openstack_project_path | length == 0
+        - not repo_status.stat.exists
+        - "'host' in _change"
+        - "'github.com' in _change.host"
+      ansible.builtin.git:
+        repo: '{{ _change.host }}/{{ _change.project }}'
+        dest: '{{ cifmw_bop_build_repo_dir }}/DLRN/data/{{ project_name_mapped.stdout }}'
+        refspec: "+refs/pull/*:refs/remotes/origin/pr/*"
+        version: 'origin/pr/{{ _change.change }}/head'
+
+    - name: "Clone Openstack {{ project_name_mapped.stdout }}" # noqa: name[template]
+      when:
+        - cifmw_bop_openstack_project_path | length == 0
+        - not repo_status.stat.exists
+        - "'host' in _change"
+        - "'opendev' in _change.host"
+      ansible.builtin.git:
+        repo: '{{ _change.host }}/{{ _change.project }}'
+        dest: '{{ cifmw_bop_build_repo_dir }}/DLRN/data/{{ project_name_mapped.stdout }}'
+        refspec: "{{ _change.refspec }}"
+        version: 'FETCH_HEAD'
+
+    - name: Find the last commit from the clonned repo
+      register: _commit
+      ansible.builtin.command:
+        cmd: git show-ref --head --hash head # noqa: command-instead-of-module
+        chdir: "{{ cifmw_bop_build_repo_dir }}/DLRN/data/{{ project_name_mapped.stdout }}"
+
+    - name: Print the last commit of the git repo
+      ansible.builtin.debug:
+        var: _commit.stdout
 
     - name: Ensure distgit repo is absent, DLRN takes care of cloning based on config
       ansible.builtin.file:

roles/pkg_build/templates/pkg_build_play.j2

@@ -13,6 +13,7 @@
     cifmw_bop_build_repo_dir: "{{ ansible_user_dir }}/{{ cifmw_project }}"
     cifmw_bop_yum_repos_dir: "{{ ansible_user_dir }}/yum.repos.d"
     cifmw_bop_gating_repo_dest: "{{ ansible_user_dir }}/gating_repo"
+    cifmw_bop_openstack_release: master
   tasks:
     - name: Gather parameters
       ansible.builtin.set_fact: