Feature branch sync - pub/build_stream to staging

.github/workflows/ansible-lint.yml

@@ -6,6 +6,7 @@ on:
       - main
       - staging
       - release_1.7.1
+      - pub/build_stream
       - pub/v2.1_rc1
       - pub/q1_dev
 

.github/workflows/pylint.yml

@@ -6,6 +6,7 @@ on:
       - main
       - staging
       - release_1.7.1
+      - pub/build_stream
       - pub/v2.1_rc1
       - pub/q1_dev
 
@@ -28,7 +29,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install ansible pylint kubernetes prettytable requests passlib
+          pip install ansible pylint kubernetes prettytable requests passlib fastapi uvicorn sqlalchemy pytest httpx argon2-cffi pyyaml dependency-injector
 
       - name: Get changed Python files (excluding deleted)
         id: changed-files
@@ -57,7 +58,9 @@ jobs:
           FILES=$(echo "${{ steps.changed-files.outputs.files }}" | tr ' ' '\n' | grep -v '^discovery/roles/telemetry/files/nersc-ldms-aggr/' | xargs)
 
           if [ -n "$FILES" ]; then
-            pylint $FILES --fail-under=${PYLINT_THRESHOLD}
+            # Set PYTHONPATH to include build_stream directory for proper import resolution
+            # This allows pylint to resolve both relative imports in build_stream and regular imports elsewhere
+            PYTHONPATH=.:./build_stream pylint $FILES --fail-under=${PYLINT_THRESHOLD}
           else
             echo "No files to lint after filtering."
           fi

.gitignore

@@ -1,3 +1,4 @@
 /.idea/
 /docs/build/
-**/__pycache__/
+**/__pycache__/
+.venv

build_image_aarch64/build_image_aarch64.yml

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -41,6 +41,44 @@
     openchami_vars_suppport: true
     omnia_metadata_support: true
 
+- name: Load build_stream configuration
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Include build_stream config file
+      ansible.builtin.include_vars:
+        file: "{{ input_project_dir }}/build_stream_config.yml"
+      failed_when: false
+
+    - name: Set build_stream variables from extra_vars
+      ansible.builtin.set_fact:
+        build_stream_job_id: "{{ job_id | default('') }}"
+        build_stream_image_key: "{{ image_key | default('') }}"
+        build_stream_functional_groups: "{{ functional_groups | default([]) }}"
+        enable_build_stream_flag: "{{ enable_build_stream | default(false) | bool }}"
+
+    - name: Debug - Show build_stream variables
+      ansible.builtin.debug:
+        msg:
+          - "build_stream_job_id: {{ build_stream_job_id }}"
+          - "build_stream_image_key: {{ build_stream_image_key }}"
+          - "build_stream_functional_groups: {{ build_stream_functional_groups }}"
+          - "enable_build_stream_flag: {{ enable_build_stream_flag }}"
+        verbosity: 2
+
+    - name: Fetch build_stream prerequisites
+      ansible.builtin.include_role:
+        name: fetch_packages
+        tasks_from: build_stream_prerequisite.yml
+      vars:
+        job_id: "{{ build_stream_job_id }}"
+        image_key: "{{ build_stream_image_key }}"
+        functional_groups: "{{ build_stream_functional_groups }}"
+        enable_build_stream: "{{ enable_build_stream_flag }}"
+      when: enable_build_stream_flag
+
 - name: Gather OIM data
   hosts: localhost
   gather_facts: false
@@ -66,9 +104,10 @@
       vars:
         oim_node_name: "{{ hostvars['localhost']['oim_node_name'] }}"
 
-- name: Generate functional groups configuration
+- name: Generate functional groups configuration when enable_build_stream is false
   ansible.builtin.import_playbook: ../utils/generate_functional_groups.yml
   tags: always
+  when: not enable_build_stream
 
 - name: Verify aarch64 functional_group presnt
   hosts: localhost
@@ -78,6 +117,7 @@
       ansible.builtin.include_role:
         name: fetch_packages
         tasks_from: check_aarch64_fg.yml
+      when: not enable_build_stream
 
 - name: Prepare aarch64 nodes
   hosts: admin_aarch64

build_image_aarch64/roles/fetch_packages/tasks/build_stream_prerequisite.yml

@@ -0,0 +1,41 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Debug - Show explicitly passed variables
+  ansible.builtin.debug:
+    msg:
+      - "job_id: {{ job_id | default('NOT_SET') }}"
+      - "image_key: {{ image_key | default('NOT_SET') }}"
+      - "functional_groups: {{ functional_groups | default('NOT_SET') }}"
+      - "enable_build_stream: {{ enable_build_stream | default('NOT_SET') }}"
+    verbosity: 2
+
+- name: Set build_stream variables from explicitly passed values
+  ansible.builtin.set_fact:
+    build_stream_job_id: "{{ job_id }}"
+    image_key: "{{ image_key }}"
+    cacheable: true
+
+- name: Normalize functional_groups input into list
+  ansible.builtin.set_fact:
+    functional_group_list: "{{ functional_groups if functional_groups is iterable and functional_groups is not string else (functional_groups | from_yaml) }}"
+  when: functional_groups is defined and enable_build_stream
+
+- name: Fail when build stream enabled without job id or functional groups
+  ansible.builtin.fail:
+    msg: "{{ build_stream_prerequisite_fail_msg }}"
+  when:
+    - enable_build_stream | bool
+    - (build_stream_job_id | default('') | string) | length == 0 or (functional_group_list | default([]) | length == 0) or (image_key | default('') | string) | length == 0   # noqa: yaml[line-length]

build_image_aarch64/roles/fetch_packages/tasks/fetch_packages.yml

@@ -14,7 +14,6 @@
 ---
 
 - name: Fetch aarch64 default_packages.json and additional_packages.json software packages
-  when: fg_aarch64
   block:
     - name: Collect base image RPM packages (default + additional + admin_debug)
       base_image_package_collector:
@@ -24,7 +23,7 @@
         software_config_path: "{{ software_config_file_path }}"
       register: base_image_output
 
-    - name: Set aarch_64_base_image_packages
+    - name: Set aarch64_base_image_packages
       ansible.builtin.set_fact:
         aarch64_base_image_packages: "{{ base_image_output.base_image_packages }}"
 
@@ -33,9 +32,25 @@
         var: aarch64_base_image_packages
         verbosity: 2
 
+    - name: Parse functional_group_config.yml to list
+      functional_group_parser:
+        functional_groups_file: "{{ functional_groups_file_path }}"
+      register: functional_group_parser_list
+      when: not enable_build_stream
+
+    - name: Set fact for functional_group_list
+      ansible.builtin.set_fact:
+        functional_group_list: "{{ functional_group_parser_list.functional_groups }}"
+      when: not enable_build_stream
+
+    - name: Debug full functional group parser output
+      ansible.builtin.debug:
+        var: functional_group_list
+        verbosity: 2
+
     - name: Read packages for compute image softwares
       image_package_collector:
-        functional_groups_file: "{{ functional_groups_file_path }}"
+        functional_groups: "{{ functional_group_list }}"
         software_config_file: "{{ software_config_file_path }}"
         input_project_dir: "{{ input_project_dir }}"
       register: compute_images_output

build_image_aarch64/roles/fetch_packages/tasks/fetch_pulp_repos.yml

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +13,7 @@
 # limitations under the License.
 ---
 
-- name: Fetch pulp endpoints only if aarch_64 functional_group present
-  when: fg_aarch64
+- name: Fetch pulp endpoints when aarch_64 build_stream enabled
   block:
     - name: Fetch pulp endpoints for aarch64
       ansible.builtin.command: >

build_image_aarch64/roles/fetch_packages/tasks/main.yml

@@ -22,16 +22,29 @@
     - name: Fail if metadata file is not present
       ansible.builtin.fail:
         msg: "{{ local_repo_check_msg }}"
+      when: not metadata_file_status.stat.exists
+
+- name: Initialize fg_aarch64 as false
+  ansible.builtin.set_fact:
+    fg_aarch64: "{{ fg_aarch64 | default(false) }}"
+  when: enable_build_stream | default(false)
 
 - name: Include functional groups config
   ansible.builtin.include_vars:
     file: "{{ functional_groups_file_path }}"
     name: functional_groups_config
+  when: not enable_build_stream
+
+- name: Set functional_groups_file_path for build_stream disabled flow
+  ansible.builtin.set_fact:
+    functional_groups_file_path: "{{ functional_groups_file_path }}"
+  when: not enable_build_stream
 
 - name: Include software config
   ansible.builtin.include_vars:
     file: "{{ software_config_file_path }}"
     name: software_config
+  when: enable_build_stream | default(false)
 
 - name: Set cluster OS facts
   ansible.builtin.set_fact:
@@ -42,6 +55,8 @@
 
 - name: Fetch pulp endpoint repos
   ansible.builtin.include_tasks: fetch_pulp_repos.yml
+  when: fg_aarch64 or enable_build_stream
 
 - name: Fetch packages for base and compute image softwares
   ansible.builtin.include_tasks: fetch_packages.yml
+  when: fg_aarch64 or enable_build_stream

build_image_aarch64/roles/fetch_packages/vars/main.yml

@@ -19,7 +19,7 @@ local_repo_check_msg: |
   Failure: metadata file is not present at path {{ metadata_file_path }}.
   Please make sure that local_repo.yml playbook is executed successfully.
 input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}"
-functional_groups_file_path: "{{ hostvars['localhost']['functional_groups_config_path'] }}"
+functional_groups_file_path: "{{ hostvars['localhost']['functional_groups_config_path'] | default('/opt/omnia/.data/functional_groups_config.yml') }}"
 software_config_file_path: "{{ input_project_dir }}/software_config.json"
 aarch64_build_image_completion_msg: |
   The playbook build_image_aarch64.yml has been completed successfully.
@@ -28,3 +28,4 @@ functional_group_absent_msg: |
   Failure: No aarch64 functional groups found in functional_group_config.yml input file.
   Please make sure aarch64 functional_group should be present in input file functional_group_config.yml
   to execute build_image_aarch64.yml successfully.
+build_stream_prerequisite_fail_msg: "Failed: enable_build_stream is set to true, make sure job_id, image_key and functional_groups were provided."

build_image_aarch64/roles/image_creation/tasks/build_base_image.yml

@@ -13,6 +13,22 @@
 #  limitations under the License.
 ---
 
+- name: Normalize build stream inputs for base image
+  ansible.builtin.set_fact:
+    enable_build_stream: "{{ enable_build_stream | default(false) | bool }}"
+    build_stream_job_id: "{{ build_stream_job_id | default('') }}"
+    image_key: "{{ image_key | default('') }}"
+    base_image_suffix: ""
+
+- name: Set base image suffix when build stream inputs present
+  ansible.builtin.set_fact:
+    base_image_suffix: "_{{ build_stream_job_id }}-{{ image_key | default('') }}"
+    rhel_base_image_name: "{{ rhel_aarch64_base_image_name }}_{{ build_stream_job_id }}-{{ image_key | default('') }}"
+  when:
+    - enable_build_stream | bool
+    - (build_stream_job_id | default('') | length) > 0
+    - (image_key | default('') | length) > 0
+
 - name: Create temporary inventory with ochami group
   ansible.builtin.copy:
     dest: "{{ aarch64_inventory_file }}"
@@ -70,4 +86,6 @@
     - name: Set openchami SELinux context
       ansible.builtin.command: chcon -R system_u:object_r:container_file_t:s0 "{{ oim_shared_path }}/omnia/openchami"
       changed_when: true
+      delegate_to: oim
+      connection: ssh
       failed_when: false

build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,6 +13,21 @@
 #  limitations under the License.
 ---
 
+- name: Normalize build stream inputs
+  ansible.builtin.set_fact:
+    enable_build_stream: "{{ enable_build_stream | default(false) | bool }}"
+    build_stream_job_id: "{{ build_stream_job_id | default('') }}"
+    image_key: "{{ image_key | default('') }}"
+    compute_image_suffix: ""
+
+- name: Set compute image suffix when build stream inputs present
+  ansible.builtin.set_fact:
+    compute_image_suffix: "_{{ build_stream_job_id }}-{{ image_key | default('') }}"
+  when:
+    - enable_build_stream | bool
+    - (build_stream_job_id | default('') | length) > 0
+    - (image_key | default('') | length) > 0
+
 - name: Create temporary inventory with ochami group
   ansible.builtin.copy:
     dest: "{{ aarch64_inventory_file }}"
@@ -23,7 +38,7 @@
 
 - name: Create aarch64 compute image log files
   ansible.builtin.file:
-    path: "{{ openchami_log_dir }}/{{ item.key }}_compute_image.log"
+    path: "{{ openchami_log_dir }}/{{ item.key }}{{ compute_image_suffix }}_compute_image.log"
     state: touch
     mode: "{{ dir_permissions_644 }}"
   loop: "{{ compute_images_dict | dict2items }}"
@@ -33,12 +48,13 @@
 - name: Render compute images templates
   ansible.builtin.template:
     src: "{{ openchami_compute_image_vars_template }}"
-    dest: "{{ openchami_dir }}/{{ item.key }}_compute_images.yaml"
+    dest: "{{ openchami_dir }}/{{ item.key }}{{ compute_image_suffix }}_compute_images.yaml"
     mode: "{{ dir_permissions_644 }}"
   vars:
     functional_group: "{{ item.value.functional_group }}"
     packages: "{{ item.value.packages }}"
-    rhel_base_compute_image_name: "{{ item.key }}"
+    base_compute_image_name: "{{ item.key }}{{ compute_image_suffix }}"
+    rhel_base_compute_image_name: "rhel-{{ item.key }}{{ compute_image_suffix }}"
   loop: "{{ compute_images_dict | dict2items }}"
   loop_control:
     loop_var: item
@@ -48,9 +64,9 @@
     set -o pipefail
     ansible-playbook {{ openchami_clone_path }}/dell/podman-quadlets/image.yaml \
     -i {{ aarch64_inventory_file }} -v \
-      --extra-vars '@{{ openchami_dir }}/{{ item.key }}_compute_images.yaml' \
+      --extra-vars '@{{ openchami_dir }}/{{ item.key }}{{ compute_image_suffix }}_compute_images.yaml' \
       --tags compute_image -v | \
-      /usr/bin/tee '{{ openchami_log_dir }}/{{ item.key }}_compute_image.log'
+      /usr/bin/tee '{{ openchami_log_dir }}/{{ item.key }}{{ compute_image_suffix }}_compute_image.log'
   async: 3600   # Set async timeout (e.g., 1 hour)
   poll: 0   # Non-blocking (continue the playbook without waiting for completion)
   loop: "{{ compute_images_dict | dict2items }}"
@@ -110,7 +126,9 @@
 
     - name: Add log file paths to message
       ansible.builtin.set_fact:
-        failure_msg_list: "{{ failure_msg_list + ['  - ' + openchami_log_dir + '/' + item + '_compute_image.log'] }}"
+        failure_msg_list: "{{ failure_msg_list + ['  - ' + openchami_log_dir + '/' + item + log_suffix + '_compute_image.log'] }}"
+      vars:
+        log_suffix: "{{ compute_image_suffix }}"
       loop: "{{ failed_images | default([]) }}"
 
     - name: Display aarch64 compute image build failure details
@@ -124,7 +142,7 @@
   always:
     - name: Remove generated compute images templates
       ansible.builtin.file:
-        path: "{{ openchami_dir }}/{{ item.key }}_compute_images.yaml"
+        path: "{{ openchami_dir }}/{{ item.key }}{{ compute_image_suffix }}_compute_images.yaml"
         state: absent
       loop: "{{ compute_images_dict | dict2items }}"
       loop_control:
@@ -138,4 +156,6 @@
     - name: Set openchami SELinux context
       ansible.builtin.command: chcon -R system_u:object_r:container_file_t:s0 "{{ oim_shared_path }}/omnia/openchami"
       changed_when: true
+      delegate_to: oim
+      connection: ssh
       failed_when: false

build_image_aarch64/roles/image_creation/templates/base_image_template.j2

@@ -4,6 +4,7 @@ rhel_base_image_name: "{{ rhel_aarch64_base_image_name }}"
 rhel_base_image: "{{ oim_node_name }}/{{ rhel_aarch64_base_image_name }}"
 cluster_name: "{{ oim_node_name }}"
 cluster_domain: "{{ domain_name }}"
+group_name: base
 rhel_base_mounts: {{ ochami_mounts | join(' ') }}
 image_build_name: {{ ochami_aarch64_image | join(' ') }}
 rhel_base_command_options: {{ ochami_base_command | join(' ') }}