From fdfd60f1dfcf33582478364d8dbbf55ea812d378 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 5 Aug 2022 09:23:57 +0100
Subject: [PATCH 01/16] CI: Add overcloud-host-image-build workflow

---
 .../workflows/overcloud-host-image-build.yml  | 141 ++++++++++++++++++
 .../environments/ci-builder/overcloud-dib.yml |  20 +++
 2 files changed, 161 insertions(+)
 create mode 100644 .github/workflows/overcloud-host-image-build.yml
 create mode 100644 etc/kayobe/environments/ci-builder/overcloud-dib.yml

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
new file mode 100644
index 000000000..d9469acdf
--- /dev/null
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -0,0 +1,141 @@
+---
+name: Build overcloud host images
+on:
+  workflow_dispatch:
+    inputs:
+      centos:
+        description: Build CentOS Stream
+        type: boolean
+        default: true
+      rocky:
+        description: Build Rocky Linux
+        type: boolean
+        default: true
+      ubuntu:
+        description: Build Ubuntu
+        type: boolean
+        default: true
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+jobs:
+  overcloud-host-image-build:
+    name: Build overcloud host images
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: src/kayobe-config
+
+      - name: Clone StackHPC Kayobe repository
+        uses: actions/checkout@v3
+        with:
+          repository: stackhpc/kayobe
+          ref: refs/heads/stackhpc/wallaby
+          path: src/kayobe
+
+      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
+      - name: Install dependencies
+        run: |
+          sudo dnf -y install python3-virtualenv
+
+      - name: Setup networking
+        run: |
+          if ! ip l show breth1 >/dev/null 2>&1; then
+              sudo ip l add breth1 type bridge
+          fi
+          sudo ip l set breth1 up
+          if ! ip a show breth1 | grep 192.168.33.3/24; then
+              sudo ip a add 192.168.33.3/24 dev breth1
+          fi
+          if ! ip l show dummy1 >/dev/null 2>&1; then
+              sudo ip l add dummy1 type dummy
+          fi
+          sudo ip l set dummy1 up
+          sudo ip l set dummy1 master breth1
+
+      - name: Install Kayobe
+        run: |
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install ../src/kayobe
+
+      - name: Bootstrap the control host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe control host bootstrap
+
+      - name: Configure the seed host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host configure
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Create bifrost_httpboot Docker volume
+        run: |
+          if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then
+            sudo docker volume create bifrost_httpboot
+          fi
+
+      - name: Build a CentOS Stream 8 overcloud host image
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe overcloud host image build --force-rebuild -e os_distribution=centos -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.centos
+
+      - name: Upload CentOS Stream 8 overcloud host image artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: CentOS Stream 8 overcloud host image
+          path: /opt/kayobe/images/overcloud-centos-stream-8
+          retention-days: 7
+        if: inputs.centos
+
+      - name: Build a Rocky Linux 8 overcloud host image
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe overcloud host image build --force-rebuild -e os_distribution=rocky -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.rocky
+
+      - name: Upload Rocky Linux 8 overcloud host image artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Rocky Linux 8 overcloud host image
+          path: /opt/kayobe/images/overcloud-rocky-linux-8
+          retention-days: 7
+        if: inputs.rocky
+
+      - name: Build a Ubuntu Focal 20.04 overcloud host image
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe overcloud host image build --force-rebuild -e os_distribution=ubuntu -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.ubuntu
+
+      - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Ubuntu Focal 20.04 overcloud host image
+          path: /opt/kayobe/images/overcloud-ubuntu-focal
+          retention-days: 7
+        if: inputs.ubuntu
+
+      - name: Clean up old images
+        run: |
+          sudo rm -rf /opt/kayobe/images/
+        if: always()
diff --git a/etc/kayobe/environments/ci-builder/overcloud-dib.yml b/etc/kayobe/environments/ci-builder/overcloud-dib.yml
new file mode 100644
index 000000000..6a8f3b0f1
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/overcloud-dib.yml
@@ -0,0 +1,20 @@
+---
+# Overcloud host disk image configuration.
+
+###############################################################################
+# Diskimage-builder configuration for overcloud host disk images.
+
+# Whether to build host disk images with DIB directly instead of through
+# Bifrost. Setting it to true disables Bifrost image build and allows images to
+# be built with the `kayobe overcloud host image build` command. Default value
+# is False. This will change in a future release.
+overcloud_dib_build_host_images: true
+
+# List of overcloud host disk images to build. Each element is a dict defining
+# an image in a format accepted by the stackhpc.os-images role. Default is to
+# build an image named "deployment_image" configured with the overcloud_dib_*
+# variables defined below: {"name": "deployment_image", "elements": "{{
+# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}",
+# "packages": "{{ overcloud_dib_packages }}"}.
+overcloud_dib_host_images:
+  - "{{ stackhpc_overcloud_dib_host_image }}"

From 6eabe77dff2da1d6971eeea607238dcfcf12dd61 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 5 Aug 2022 15:03:44 +0100
Subject: [PATCH 02/16] CI: Workaround lack of internal DNS in SMS lab

Use an etc-hosts DIB element to add the test Pulp server to /etc/hosts
in the image.
---
 .github/workflows/overcloud-host-image-build.yml | 6 ++++++
 etc/kayobe/overcloud-dib.yml                     | 3 ++-
 etc/kayobe/stackhpc-overcloud-dib.yml            | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index d9469acdf..e93e88205 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -118,6 +118,12 @@ jobs:
           retention-days: 7
         if: inputs.rocky
 
+      # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
+      # Do this via config?
+      - name: Enable EPEL repository
+        run: |
+          sudo dnf config-manager --set-enabled epel
+
       - name: Build a Ubuntu Focal 20.04 overcloud host image
         run: |
           source venvs/kayobe/bin/activate &&
diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml
index 9c34650ff..c9fee7405 100644
--- a/etc/kayobe/overcloud-dib.yml
+++ b/etc/kayobe/overcloud-dib.yml
@@ -71,7 +71,8 @@ overcloud_dib_host_packages_extra:
 overcloud_dib_git_elements_extra:
   - repo: "https://github.com/stackhpc/stackhpc-image-elements"
     local: "{{ source_checkout_path }}/stackhpc-image-elements"
-    version: "v1.4.0"
+    # FIXME:
+    version: "etc-hosts"
     elements_path: "elements"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See
diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml
index 84fffbc0f..4925e36e9 100644
--- a/etc/kayobe/stackhpc-overcloud-dib.yml
+++ b/etc/kayobe/stackhpc-overcloud-dib.yml
@@ -25,6 +25,7 @@ stackhpc_overcloud_dib_elements:
   - "cloud-init-datasources"
   - "{% if os_distribution in ['centos', 'rocky'] %}disable-selinux{% endif %}"
   - "enable-serial-console"
+  - "{% if kayobe_environment == 'ci-builder' %}etc-hosts{% endif %}"
   - "vm"
   - "block-device-efi"
   - "cloud-init"
@@ -45,6 +46,9 @@ stackhpc_overcloud_dib_env_vars:
   # sometimes).
   # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}"
   DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}"
+  # FIXME: Workaround for lack of internal DNS on SMS lab.
+  DIB_ETC_HOSTS: |
+    10.205.3.187 pulp-server pulp-server.internal.sms-cloud
   DIB_RELEASE: "{{ overcloud_dib_os_release }}"
   DIB_SUDOERS_FILENAME: "no-fqdn"
   # Avoid DNS queries during sudo commands, since we might not always have working DNS.

From 65a3b593566e2a510bd1518d6c865a7e999c8cf5 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 9 Dec 2022 16:30:31 +0000
Subject: [PATCH 03/16] Draft layout for pulp image hosting

---
 .../workflows/overcloud-host-image-build.yml  |  33 +++---
 etc/kayobe/ansible/upload-image.yml           | 102 ++++++++++++++++++
 2 files changed, 120 insertions(+), 15 deletions(-)
 create mode 100644 etc/kayobe/ansible/upload-image.yml

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index e93e88205..a47098197 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -94,11 +94,12 @@ jobs:
         if: inputs.centos
 
       - name: Upload CentOS Stream 8 overcloud host image artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: CentOS Stream 8 overcloud host image
-          path: /opt/kayobe/images/overcloud-centos-stream-8
-          retention-days: 7
+        run: |
+          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          -i localhost \
+          -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
+          -e os_distribution='centos' \
+          -e os_version='stream8'
         if: inputs.centos
 
       - name: Build a Rocky Linux 8 overcloud host image
@@ -111,11 +112,12 @@ jobs:
         if: inputs.rocky
 
       - name: Upload Rocky Linux 8 overcloud host image artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: Rocky Linux 8 overcloud host image
-          path: /opt/kayobe/images/overcloud-rocky-linux-8
-          retention-days: 7
+        run: |
+          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          -i localhost \
+          -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
+          -e os_distribution='rocky' \
+          -e os_version='8'
         if: inputs.rocky
 
       # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
@@ -134,11 +136,12 @@ jobs:
         if: inputs.ubuntu
 
       - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: Ubuntu Focal 20.04 overcloud host image
-          path: /opt/kayobe/images/overcloud-ubuntu-focal
-          retention-days: 7
+        run: |
+          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          -i localhost \
+          -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
+          -e os_distribution='ubuntu' \
+          -e os_version='20.04-focal'
         if: inputs.ubuntu
 
       - name: Clean up old images
diff --git a/etc/kayobe/ansible/upload-image.yml b/etc/kayobe/ansible/upload-image.yml
new file mode 100644
index 000000000..9ef3cb9f6
--- /dev/null
+++ b/etc/kayobe/ansible/upload-image.yml
@@ -0,0 +1,102 @@
+---
+- name: Upload and create a distribution for an image
+  hosts: localhost
+  # These vars are placeholders, they should be deleted before merging
+  # ==================================================================
+  vars:
+    pulp_username: username
+    pulp_password: password
+    pulp_url: http://example.com
+    os_distribution: ubuntu
+    os_version: 20.04-focal
+    image_path: path/to/ubuntu.iso
+  # ==================================================================
+  tasks:
+    - name: Upload an artifact
+      pulp.squeezer.artifact:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        file: "{{ image_path }}"
+        state: present
+
+    - name: Get sha256 hash
+      ansible.builtin.stat:
+        path: "{{ image_path }}"
+        checksum_algorithm: sha256
+      register: file_stats
+
+    - name: Create file content from artifact
+      pulp.squeezer.file_content:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        sha256: "{{ file_stats.stat.checksum }}"
+        relative_path: "{{ os_distribution }}.img"
+        state: present
+
+    - name: Create a file repo
+      pulp.squeezer.file_repository:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        name: "{{ os_distribution }}_img_repo"
+        state: present
+
+    - name: Add content to repo
+      pulp.squeezer.file_repository_content:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        repository: "{{ os_distribution }}_img_repo"
+        present_content:
+          - relative_path: "{{ os_distribution }}.img"
+            sha256: "{{ file_stats.stat.checksum }}"
+      register: repo_details
+
+    # Squeezer wants the version as an int. The previous play only returns the
+    # href of the latest version.
+    # The version can be returned by querying that href, however the last
+    # number in the href is the version. regex_search is used to extract it
+    - name: Create a new publication to point to this version
+      pulp.squeezer.file_publication:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        repository: "{{ os_distribution }}_img_repo"
+        state: present
+        version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
+      register: version_publication_details
+
+    - name: Create a publication to point to the latest version
+      pulp.squeezer.file_publication:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        repository: "{{ os_distribution }}_img_repo"
+        state: present
+      register: latest_publication_details
+
+    - name: Create distribution for latest version
+      pulp.squeezer.file_distribution:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        name: "{{ os_distribution }}_{{ os_version }}_latest"
+        base_path: "images/{{ os_distribution }}/{{ os_version }}/latest"
+        publication: "{{ latest_publication_details.publication.pulp_href }}"
+        content_guard: release
+        state: present
+      register: latest_distribution_details
+
+    - name: Create distribution for given version
+      pulp.squeezer.file_distribution:
+        pulp_url: "{{ pulp_url }}"
+        username: "{{ pulp_username }}"
+        password: "{{ pulp_password }}"
+        name: "{{ os_distribution }}_{{ os_version }}_{{ ansible_date_time.iso8601_basic_short }}"
+        base_path: "images/{{ os_distribution }}/{{ os_version }}/{{ ansible_date_time.iso8601_basic_short }}"
+        publication: "{{ version_publication_details.publication.pulp_href }}"
+        content_guard: release
+        state: present
+      when: latest_distribution_details.changed is true

From d9eec6b68252fb59b9f59637dd33fc42486c34a1 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 13 Dec 2022 10:50:06 +0000
Subject: [PATCH 04/16] upload-image removing placeholder vars

---
 .../workflows/overcloud-host-image-build.yml  | 15 +++-
 etc/kayobe/ansible/requirements.yml           |  2 +-
 etc/kayobe/ansible/upload-image.yml           | 82 +++++++++----------
 .../environments/ci-builder/stackhpc-ci.yml   |  3 +
 4 files changed, 54 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index a47098197..5cb52645c 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -95,11 +95,14 @@ jobs:
 
       - name: Upload CentOS Stream 8 overcloud host image artifact
         run: |
-          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
           -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
           -e os_distribution='centos' \
           -e os_version='stream8'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.centos
 
       - name: Build a Rocky Linux 8 overcloud host image
@@ -113,11 +116,14 @@ jobs:
 
       - name: Upload Rocky Linux 8 overcloud host image artifact
         run: |
-          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
           -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
           -e os_distribution='rocky' \
           -e os_version='8'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky
 
       # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
@@ -137,11 +143,14 @@ jobs:
 
       - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
         run: |
-          ansible-playbook src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
           -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
           -e os_distribution='ubuntu' \
           -e os_version='20.04-focal'
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu
 
       - name: Clean up old images
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 77aaa9f99..87c6b4bb6 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -3,6 +3,6 @@ collections:
   - name: stackhpc.cephadm
     version: 1.10.0
   - name: stackhpc.pulp
-    version: 0.4.1
+    version: 0.4.2
 roles:
   - src: stackhpc.vxlan
diff --git a/etc/kayobe/ansible/upload-image.yml b/etc/kayobe/ansible/upload-image.yml
index 9ef3cb9f6..0afc86483 100644
--- a/etc/kayobe/ansible/upload-image.yml
+++ b/etc/kayobe/ansible/upload-image.yml
@@ -1,22 +1,16 @@
 ---
 - name: Upload and create a distribution for an image
   hosts: localhost
-  # These vars are placeholders, they should be deleted before merging
-  # ==================================================================
   vars:
-    pulp_username: username
-    pulp_password: password
-    pulp_url: http://example.com
-    os_distribution: ubuntu
-    os_version: 20.04-focal
-    image_path: path/to/ubuntu.iso
-  # ==================================================================
+    remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
+    remote_pulp_username: "{{ stackhpc_image_repositiory_username }}"
+    remote_pulp_password: "{{ stackhpc_image_repositiory_password }}"
   tasks:
     - name: Upload an artifact
       pulp.squeezer.artifact:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
         file: "{{ image_path }}"
         state: present
 
@@ -28,29 +22,29 @@
 
     - name: Create file content from artifact
       pulp.squeezer.file_content:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
         sha256: "{{ file_stats.stat.checksum }}"
-        relative_path: "{{ os_distribution }}.img"
+        relative_path: "{{ os_distribution }}-{{ os_version }}.img"
         state: present
 
-    - name: Create a file repo
+    - name: Ensure file repo exists
       pulp.squeezer.file_repository:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
-        name: "{{ os_distribution }}_img_repo"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        name: "{{ os_distribution }}_{{ os_version }}_img_repo"
         state: present
 
-    - name: Add content to repo
+    - name: Add content to file repo
       pulp.squeezer.file_repository_content:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
-        repository: "{{ os_distribution }}_img_repo"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
         present_content:
-          - relative_path: "{{ os_distribution }}.img"
+          - relative_path: "{{ os_distribution }}-{{ os_version }}.img"
             sha256: "{{ file_stats.stat.checksum }}"
       register: repo_details
 
@@ -60,43 +54,43 @@
     # number in the href is the version. regex_search is used to extract it
     - name: Create a new publication to point to this version
       pulp.squeezer.file_publication:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
-        repository: "{{ os_distribution }}_img_repo"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
         state: present
         version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
       register: version_publication_details
 
     - name: Create a publication to point to the latest version
       pulp.squeezer.file_publication:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
-        repository: "{{ os_distribution }}_img_repo"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
         state: present
       register: latest_publication_details
 
-    - name: Create distribution for latest version
+    - name: Update distribution for latest version
       pulp.squeezer.file_distribution:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
         name: "{{ os_distribution }}_{{ os_version }}_latest"
         base_path: "images/{{ os_distribution }}/{{ os_version }}/latest"
         publication: "{{ latest_publication_details.publication.pulp_href }}"
-        content_guard: release
+        content_guard: development
         state: present
       register: latest_distribution_details
 
     - name: Create distribution for given version
       pulp.squeezer.file_distribution:
-        pulp_url: "{{ pulp_url }}"
-        username: "{{ pulp_username }}"
-        password: "{{ pulp_password }}"
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
         name: "{{ os_distribution }}_{{ os_version }}_{{ ansible_date_time.iso8601_basic_short }}"
         base_path: "images/{{ os_distribution }}/{{ os_version }}/{{ ansible_date_time.iso8601_basic_short }}"
         publication: "{{ version_publication_details.publication.pulp_href }}"
-        content_guard: release
+        content_guard: development
         state: present
       when: latest_distribution_details.changed is true
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 4a9e50f1c..e9903c978 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -82,3 +82,6 @@ stackhpc_docker_registry_password: !vault |
   34386133383366326639353432386235336132663839333337323739633434613934346462363031
   3265323831663964360a643962346231386462323236373963633066393736323234303833363535
   3664
+
+stackhpc_image_repositiory_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_image_repositiory_password: "{{ stackhpc_docker_registry_password }}"

From 9b0ffca519bf57b3b9e5d818b51ae5ca4fa477e0 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 13 Dec 2022 15:44:14 +0000
Subject: [PATCH 05/16] Renaming, and various fixes for, pulp_upload_image

---
 .../workflows/overcloud-host-image-build.yml  | 24 ++++++-----
 ...upload-image.yml => pulp_upload_image.yml} | 43 ++++++++++++++-----
 etc/kayobe/ansible/requirements.yml           |  2 +-
 .../environments/ci-builder/stackhpc-ci.yml   |  4 +-
 4 files changed, 49 insertions(+), 24 deletions(-)
 rename etc/kayobe/ansible/{upload-image.yml => pulp_upload_image.yml} (68%)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 5cb52645c..5547079ea 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -15,6 +15,9 @@ on:
         description: Build Ubuntu
         type: boolean
         default: true
+  push:
+    branches:
+      - pulp-image-hosting
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -32,7 +35,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/wallaby
+          ref: refs/heads/stackhpc/xena
           path: src/kayobe
 
       # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
@@ -97,10 +100,8 @@ jobs:
         run: |
           kayobe playbook run \
           src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
-          -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
-          -e os_distribution='centos' \
-          -e os_version='stream8'
+          -e os_distribution='centos'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.centos
@@ -118,10 +119,8 @@ jobs:
         run: |
           kayobe playbook run \
           src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
-          -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
-          -e os_distribution='rocky' \
-          -e os_version='8'
+          -e os_distribution='rocky'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky
@@ -145,14 +144,19 @@ jobs:
         run: |
           kayobe playbook run \
           src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
-          -i localhost \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
-          -e os_distribution='ubuntu' \
-          -e os_version='20.04-focal'
+          -e os_distribution='ubuntu'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu
 
+      - name: Upload updated images artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Updated images list
+          path: /tmp/updated_images.txt
+          retention-days: 7
+
       - name: Clean up old images
         run: |
           sudo rm -rf /opt/kayobe/images/
diff --git a/etc/kayobe/ansible/upload-image.yml b/etc/kayobe/ansible/pulp_upload_image.yml
similarity index 68%
rename from etc/kayobe/ansible/upload-image.yml
rename to etc/kayobe/ansible/pulp_upload_image.yml
index 0afc86483..2e30bf051 100644
--- a/etc/kayobe/ansible/upload-image.yml
+++ b/etc/kayobe/ansible/pulp_upload_image.yml
@@ -3,15 +3,22 @@
   hosts: localhost
   vars:
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
-    remote_pulp_username: "{{ stackhpc_image_repositiory_username }}"
-    remote_pulp_password: "{{ stackhpc_image_repositiory_password }}"
+    remote_pulp_username: "{{ stackhpc_image_repository_username }}"
+    remote_pulp_password: "{{ stackhpc_image_repository_password }}"
+    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os-version }}"
   tasks:
+    - name: Get filename
+      find:
+        paths: "{{ image_path }}"
+        patterns: '*.qcow2'
+      register: found_files
+
     - name: Upload an artifact
       pulp.squeezer.artifact:
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        file: "{{ image_path }}"
+        file: "{{ found_files[0].path }}"
         state: present
 
     - name: Get sha256 hash
@@ -26,7 +33,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         sha256: "{{ file_stats.stat.checksum }}"
-        relative_path: "{{ os_distribution }}-{{ os_version }}.img"
+        relative_path: "{{ found_files[0].path | basename }}"
         state: present
 
     - name: Ensure file repo exists
@@ -34,7 +41,7 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ os_distribution }}_{{ os_version }}_img_repo"
+        name: "{{ repository_name }}"
         state: present
 
     - name: Add content to file repo
@@ -42,9 +49,9 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
+        repository: "{{ repository_name }}"
         present_content:
-          - relative_path: "{{ os_distribution }}-{{ os_version }}.img"
+          - relative_path: "{{ found_files[0].path | basename }}"
             sha256: "{{ file_stats.stat.checksum }}"
       register: repo_details
 
@@ -57,7 +64,7 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
+        repository: "{{ repository_name }}"
         state: present
         version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
       register: version_publication_details
@@ -67,7 +74,7 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        repository: "{{ os_distribution }}_{{ os_version }}_img_repo"
+        repository: "{{ repository_name }}"
         state: present
       register: latest_publication_details
 
@@ -76,7 +83,7 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ os_distribution }}_{{ os_version }}_latest"
+        name: "{{ repository_name }}_latest"
         base_path: "images/{{ os_distribution }}/{{ os_version }}/latest"
         publication: "{{ latest_publication_details.publication.pulp_href }}"
         content_guard: development
@@ -88,9 +95,23 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ os_distribution }}_{{ os_version }}_{{ ansible_date_time.iso8601_basic_short }}"
+        name: "{{ repository_name }}_{{ ansible_date_time.iso8601_basic_short }}"
         base_path: "images/{{ os_distribution }}/{{ os_version }}/{{ ansible_date_time.iso8601_basic_short }}"
         publication: "{{ version_publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
       when: latest_distribution_details.changed is true
+
+    - name: Update new images file with versioned path
+      lineinfile:
+        path: /tmp/updated_images.txt
+        line: "{{ remote_pulp_url }}pulp/content/images/{{ os_distribution }}/{{ os_version }}/\
+        {{ ansible_date_time.iso8601_basic_short }}/{{ found_files[0].path | basename }}"
+      when: latest_distribution_details.changed is true
+
+    - name: Update new images file with latest path
+      lineinfile:
+        path: /tmp/updated_images.txt
+        line: "{{ remote_pulp_url }}pulp/content/images/{{ os_distribution }}/{{ os_version }}/\
+        latest/{{ found_files[0].path | basename }}"
+      when: latest_distribution_details.changed is true
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 87c6b4bb6..77aaa9f99 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -3,6 +3,6 @@ collections:
   - name: stackhpc.cephadm
     version: 1.10.0
   - name: stackhpc.pulp
-    version: 0.4.2
+    version: 0.4.1
 roles:
   - src: stackhpc.vxlan
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index e9903c978..13359f72f 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -83,5 +83,5 @@ stackhpc_docker_registry_password: !vault |
   3265323831663964360a643962346231386462323236373963633066393736323234303833363535
   3664
 
-stackhpc_image_repositiory_username: "{{ stackhpc_docker_registry_username }}"
-stackhpc_image_repositiory_password: "{{ stackhpc_docker_registry_password }}"
+stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"

From 7cbc36fcd4cb61ff08bf9d9257e2a364755f8eec Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 13 Dec 2022 16:47:33 +0000
Subject: [PATCH 06/16] Image upload functioning with workarounds

---
 .../workflows/overcloud-host-image-build.yml  | 13 +++---
 etc/kayobe/ansible/pulp_upload_image.yml      | 42 +++++++++++++------
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 5547079ea..91907c817 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -98,13 +98,14 @@ jobs:
 
       - name: Upload CentOS Stream 8 overcloud host image artifact
         run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
           -e os_distribution='centos'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.centos
 
       - name: Build a Rocky Linux 8 overcloud host image
         run: |
@@ -117,8 +118,8 @@ jobs:
 
       - name: Upload Rocky Linux 8 overcloud host image artifact
         run: |
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          source src/kayobe-config/.automation/pipeline/playbook-run.sh \
+          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
           -e os_distribution='rocky'
         env:
@@ -142,8 +143,8 @@ jobs:
 
       - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
         run: |
-          kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/upload-image.yml \
+          source src/kayobe-config/.automation/pipeline/playbook-run.sh \
+          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
           -e os_distribution='ubuntu'
         env:
diff --git a/etc/kayobe/ansible/pulp_upload_image.yml b/etc/kayobe/ansible/pulp_upload_image.yml
index 2e30bf051..7a30913c0 100644
--- a/etc/kayobe/ansible/pulp_upload_image.yml
+++ b/etc/kayobe/ansible/pulp_upload_image.yml
@@ -5,27 +5,44 @@
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
     remote_pulp_username: "{{ stackhpc_image_repository_username }}"
     remote_pulp_password: "{{ stackhpc_image_repository_password }}"
-    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os-version }}"
+    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
   tasks:
+    - name: Debug - create fake image directory # TODO: remove
+      ansible.builtin.file:
+        path: "{{ image_path }}"
+        state: directory
+        mode: '0766'
+      become: true
+
+    - name: Debug - create fake image file # TODO: remove
+      copy:
+        content: This is the image file content version 6
+        dest: "{{ image_path }}/fake-image.qcow2"
+        mode: '0766'
+      become: true
+
     - name: Get filename
       find:
         paths: "{{ image_path }}"
         patterns: '*.qcow2'
       register: found_files
+      become: true # TODO: remove
 
     - name: Upload an artifact
       pulp.squeezer.artifact:
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        file: "{{ found_files[0].path }}"
+        file: "{{ found_files.files[0].path }}"
         state: present
+      become: true # TODO: remove
 
     - name: Get sha256 hash
       ansible.builtin.stat:
-        path: "{{ image_path }}"
+        path: "{{ found_files.files[0].path }}"
         checksum_algorithm: sha256
       register: file_stats
+      become: true # TODO: remove
 
     - name: Create file content from artifact
       pulp.squeezer.file_content:
@@ -33,7 +50,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         sha256: "{{ file_stats.stat.checksum }}"
-        relative_path: "{{ found_files[0].path | basename }}"
+        relative_path: "{{ found_files.files[0].path | basename }}"
         state: present
 
     - name: Ensure file repo exists
@@ -51,7 +68,7 @@
         password: "{{ remote_pulp_password }}"
         repository: "{{ repository_name }}"
         present_content:
-          - relative_path: "{{ found_files[0].path | basename }}"
+          - relative_path: "{{ found_files.files[0].path | basename }}"
             sha256: "{{ file_stats.stat.checksum }}"
       register: repo_details
 
@@ -84,7 +101,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_latest"
-        base_path: "images/{{ os_distribution }}/{{ os_version }}/latest"
+        base_path: "images/{{ os_distribution }}/{{ os_release }}/latest"
         publication: "{{ latest_publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -95,8 +112,8 @@
         pulp_url: "{{ remote_pulp_url }}"
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
-        name: "{{ repository_name }}_{{ ansible_date_time.iso8601_basic_short }}"
-        base_path: "images/{{ os_distribution }}/{{ os_version }}/{{ ansible_date_time.iso8601_basic_short }}"
+        name: "{{ repository_name }}_{{ ansible_facts.date_time.iso8601_basic_short }}"
+        base_path: "images/{{ os_distribution }}/{{ os_release }}/{{ ansible_facts.date_time.iso8601_basic_short }}"
         publication: "{{ version_publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -105,13 +122,14 @@
     - name: Update new images file with versioned path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}pulp/content/images/{{ os_distribution }}/{{ os_version }}/\
-        {{ ansible_date_time.iso8601_basic_short }}/{{ found_files[0].path | basename }}"
+        line: "{{ remote_pulp_url }}/pulp/content/images/{{ os_distribution }}/{{ os_release }}/\
+        {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}"
+        create: true
       when: latest_distribution_details.changed is true
 
     - name: Update new images file with latest path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}pulp/content/images/{{ os_distribution }}/{{ os_version }}/\
-        latest/{{ found_files[0].path | basename }}"
+        line: "{{ remote_pulp_url }}/pulp/content/images/{{ os_distribution }}/{{ os_release }}/\
+        latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed is true

From fa20578695131a5c06c7f80362740c3d2474cb52 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 15 Dec 2022 10:00:24 +0000
Subject: [PATCH 07/16] renaming pulp-upload-image, removing workarounds

---
 .../workflows/overcloud-host-image-build.yml  | 24 ++++++++-----
 ...upload_image.yml => pulp-upload-image.yml} | 35 ++++++++++---------
 2 files changed, 34 insertions(+), 25 deletions(-)
 rename etc/kayobe/ansible/{pulp_upload_image.yml => pulp-upload-image.yml} (80%)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 91907c817..7bdef78f8 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -15,7 +15,7 @@ on:
         description: Build Ubuntu
         type: boolean
         default: true
-  push:
+  push: # TODO: remove
     branches:
       - pulp-image-hosting
 
@@ -87,6 +87,10 @@ jobs:
             sudo docker volume create bifrost_httpboot
           fi
 
+      - name: Clean any previous build artifact
+        run: |
+          rm -f /tmp/updated_images.txt
+
       - name: Build a CentOS Stream 8 overcloud host image
         run: |
           source venvs/kayobe/bin/activate &&
@@ -101,7 +105,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
           -e os_distribution='centos'
         env:
@@ -118,13 +122,15 @@ jobs:
 
       - name: Upload Rocky Linux 8 overcloud host image artifact
         run: |
-          source src/kayobe-config/.automation/pipeline/playbook-run.sh \
-          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
           -e os_distribution='rocky'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.rocky
+        # if: inputs.rocky
 
       # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
       # Do this via config?
@@ -143,13 +149,15 @@ jobs:
 
       - name: Upload Ubuntu Focal 20.04 overcloud host image artifact
         run: |
-          source src/kayobe-config/.automation/pipeline/playbook-run.sh \
-          src/kayobe-config/etc/kayobe/ansible/pulp_upload_image.yml \
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
           -e os_distribution='ubuntu'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.ubuntu
+        # if: inputs.ubuntu
 
       - name: Upload updated images artifact
         uses: actions/upload-artifact@v3
diff --git a/etc/kayobe/ansible/pulp_upload_image.yml b/etc/kayobe/ansible/pulp-upload-image.yml
similarity index 80%
rename from etc/kayobe/ansible/pulp_upload_image.yml
rename to etc/kayobe/ansible/pulp-upload-image.yml
index 7a30913c0..fbd428acd 100644
--- a/etc/kayobe/ansible/pulp_upload_image.yml
+++ b/etc/kayobe/ansible/pulp-upload-image.yml
@@ -6,6 +6,7 @@
     remote_pulp_username: "{{ stackhpc_image_repository_username }}"
     remote_pulp_password: "{{ stackhpc_image_repository_password }}"
     repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
+    base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
     - name: Debug - create fake image directory # TODO: remove
       ansible.builtin.file:
@@ -16,8 +17,8 @@
 
     - name: Debug - create fake image file # TODO: remove
       copy:
-        content: This is the image file content version 6
-        dest: "{{ image_path }}/fake-image.qcow2"
+        content: This is the image file content version 14
+        dest: "{{ image_path }}/{{ os_distribution }}-{{ os_release }}.qcow2"
         mode: '0766'
       become: true
 
@@ -84,16 +85,16 @@
         repository: "{{ repository_name }}"
         state: present
         version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
-      register: version_publication_details
+      register: publication_details
 
-    - name: Create a publication to point to the latest version
-      pulp.squeezer.file_publication:
-        pulp_url: "{{ remote_pulp_url }}"
-        username: "{{ remote_pulp_username }}"
-        password: "{{ remote_pulp_password }}"
-        repository: "{{ repository_name }}"
-        state: present
-      register: latest_publication_details
+    # - name: Create a publication to point to the latest version
+    #   pulp.squeezer.file_publication:
+    #     pulp_url: "{{ remote_pulp_url }}"
+    #     username: "{{ remote_pulp_username }}"
+    #     password: "{{ remote_pulp_password }}"
+    #     repository: "{{ repository_name }}"
+    #     state: present
+    #   register: publication_details
 
     - name: Update distribution for latest version
       pulp.squeezer.file_distribution:
@@ -101,8 +102,8 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_latest"
-        base_path: "images/{{ os_distribution }}/{{ os_release }}/latest"
-        publication: "{{ latest_publication_details.publication.pulp_href }}"
+        base_path: "{{ base_path }}/latest"
+        publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
       register: latest_distribution_details
@@ -113,8 +114,8 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_{{ ansible_facts.date_time.iso8601_basic_short }}"
-        base_path: "images/{{ os_distribution }}/{{ os_release }}/{{ ansible_facts.date_time.iso8601_basic_short }}"
-        publication: "{{ version_publication_details.publication.pulp_href }}"
+        base_path: "{{ base_path }}/{{ ansible_facts.date_time.iso8601_basic_short }}"
+        publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
       when: latest_distribution_details.changed is true
@@ -122,7 +123,7 @@
     - name: Update new images file with versioned path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/images/{{ os_distribution }}/{{ os_release }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}"
         create: true
       when: latest_distribution_details.changed is true
@@ -130,6 +131,6 @@
     - name: Update new images file with latest path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/images/{{ os_distribution }}/{{ os_release }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed is true

From 0f5c83f12b44470bb4a663e7597c2febfdbe0f9a Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 16 Dec 2022 09:23:03 +0000
Subject: [PATCH 08/16] Finalised pulp-upload-image and workflow

---
 .../workflows/overcloud-host-image-build.yml  | 21 +++++++++------
 etc/kayobe/ansible/pulp-upload-image.yml      | 26 -------------------
 2 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 7bdef78f8..9518428e4 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -15,9 +15,6 @@ on:
         description: Build Ubuntu
         type: boolean
         default: true
-  push: # TODO: remove
-    branches:
-      - pulp-image-hosting
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -95,7 +92,9 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild -e os_distribution=centos -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8
+          kayobe overcloud host image build --force-rebuild \
+          -e os_distribution=centos \
+          -e stackhpc_overcloud_dib_name=overcloud-centos-stream-8
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.centos
@@ -110,12 +109,15 @@ jobs:
           -e os_distribution='centos'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.centos
 
       - name: Build a Rocky Linux 8 overcloud host image
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild -e os_distribution=rocky -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8
+          kayobe overcloud host image build --force-rebuild \
+          -e os_distribution=rocky \
+          -e stackhpc_overcloud_dib_name=overcloud-rocky-linux-8
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky
@@ -130,7 +132,7 @@ jobs:
           -e os_distribution='rocky'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        # if: inputs.rocky
+        if: inputs.rocky
 
       # FIXME: Need EPEL on CentOS for debootstrap. It is disabled by default.
       # Do this via config?
@@ -142,7 +144,9 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud host image build --force-rebuild -e os_distribution=ubuntu -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
+          kayobe overcloud host image build --force-rebuild \
+          -e os_distribution=ubuntu \
+          -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu
@@ -157,7 +161,7 @@ jobs:
           -e os_distribution='ubuntu'
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        # if: inputs.ubuntu
+        if: inputs.ubuntu
 
       - name: Upload updated images artifact
         uses: actions/upload-artifact@v3
@@ -165,6 +169,7 @@ jobs:
           name: Updated images list
           path: /tmp/updated_images.txt
           retention-days: 7
+        if: always()
 
       - name: Clean up old images
         run: |
diff --git a/etc/kayobe/ansible/pulp-upload-image.yml b/etc/kayobe/ansible/pulp-upload-image.yml
index fbd428acd..41c192de0 100644
--- a/etc/kayobe/ansible/pulp-upload-image.yml
+++ b/etc/kayobe/ansible/pulp-upload-image.yml
@@ -8,26 +8,11 @@
     repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
     base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
-    - name: Debug - create fake image directory # TODO: remove
-      ansible.builtin.file:
-        path: "{{ image_path }}"
-        state: directory
-        mode: '0766'
-      become: true
-
-    - name: Debug - create fake image file # TODO: remove
-      copy:
-        content: This is the image file content version 14
-        dest: "{{ image_path }}/{{ os_distribution }}-{{ os_release }}.qcow2"
-        mode: '0766'
-      become: true
-
     - name: Get filename
       find:
         paths: "{{ image_path }}"
         patterns: '*.qcow2'
       register: found_files
-      become: true # TODO: remove
 
     - name: Upload an artifact
       pulp.squeezer.artifact:
@@ -36,14 +21,12 @@
         password: "{{ remote_pulp_password }}"
         file: "{{ found_files.files[0].path }}"
         state: present
-      become: true # TODO: remove
 
     - name: Get sha256 hash
       ansible.builtin.stat:
         path: "{{ found_files.files[0].path }}"
         checksum_algorithm: sha256
       register: file_stats
-      become: true # TODO: remove
 
     - name: Create file content from artifact
       pulp.squeezer.file_content:
@@ -87,15 +70,6 @@
         version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
       register: publication_details
 
-    # - name: Create a publication to point to the latest version
-    #   pulp.squeezer.file_publication:
-    #     pulp_url: "{{ remote_pulp_url }}"
-    #     username: "{{ remote_pulp_username }}"
-    #     password: "{{ remote_pulp_password }}"
-    #     repository: "{{ repository_name }}"
-    #     state: present
-    #   register: publication_details
-
     - name: Update distribution for latest version
       pulp.squeezer.file_distribution:
         pulp_url: "{{ remote_pulp_url }}"

From 3aa63a7f48663a2ac47d19faedda151329712113 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 16 Feb 2023 13:42:42 +0000
Subject: [PATCH 09/16] updated etc hosts workaround

---
 etc/kayobe/overcloud-dib.yml          | 3 +--
 etc/kayobe/stackhpc-overcloud-dib.yml | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml
index c9fee7405..46a02830b 100644
--- a/etc/kayobe/overcloud-dib.yml
+++ b/etc/kayobe/overcloud-dib.yml
@@ -71,8 +71,7 @@ overcloud_dib_host_packages_extra:
 overcloud_dib_git_elements_extra:
   - repo: "https://github.com/stackhpc/stackhpc-image-elements"
     local: "{{ source_checkout_path }}/stackhpc-image-elements"
-    # FIXME:
-    version: "etc-hosts"
+    version: "v1.5.0"
     elements_path: "elements"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See
diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml
index 4925e36e9..ab173b017 100644
--- a/etc/kayobe/stackhpc-overcloud-dib.yml
+++ b/etc/kayobe/stackhpc-overcloud-dib.yml
@@ -47,7 +47,7 @@ stackhpc_overcloud_dib_env_vars:
   # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}"
   DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}"
   # FIXME: Workaround for lack of internal DNS on SMS lab.
-  DIB_ETC_HOSTS: |
+  DIB_ETC_HOSTS_EXTRA: |
     10.205.3.187 pulp-server pulp-server.internal.sms-cloud
   DIB_RELEASE: "{{ overcloud_dib_os_release }}"
   DIB_SUDOERS_FILENAME: "no-fqdn"

From 6129b3bbe590a1c4de873bc880c710d9f9ba9082 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 16 Dec 2022 15:36:54 +0000
Subject: [PATCH 10/16] Adding overcloud host image sources

---
 etc/kayobe/kolla/config/bifrost/bifrost.yml | 12 ++++++++++++
 etc/kayobe/stackhpc.yml                     | 17 +++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 etc/kayobe/kolla/config/bifrost/bifrost.yml

diff --git a/etc/kayobe/kolla/config/bifrost/bifrost.yml b/etc/kayobe/kolla/config/bifrost/bifrost.yml
new file mode 100644
index 000000000..cb8ddbf3e
--- /dev/null
+++ b/etc/kayobe/kolla/config/bifrost/bifrost.yml
@@ -0,0 +1,12 @@
+---
+# Use images from Ark.
+{% if stackhpc_download_overcloud_images %}
+use_cirros: true
+{% if os_distribution == 'ubuntu' %}
+cirros_deploy_image_upstream_url: "{{ stackhpc_ubuntu_overcloud_host_image_url }}"
+{% elif os_distribution == 'rocky' %}
+cirros_deploy_image_upstream_url: "{{ stackhpc_rocky_overcloud_host_image_url }}"
+{% else %}
+cirros_deploy_image_upstream_url: "{{ stackhpc_centos_overcloud_host_image_url }}"
+{% endif %}
+{% endif %}
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 38a3efded..52b402adc 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -130,6 +130,23 @@ stackhpc_kolla_source_version: "stackhpc/{{ openstack_release }}"
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
 stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
 
+###############################################################################
+# Overcloud host image source
+
+stackhpc_download_overcloud_images: false
+
+# Ubuntu overcloud host image source
+stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image }}/overcloud-ubuntu-focal.qcow2"
+stackhpc_ubuntu_overcloud_host_image_version: "latest"
+
+# Rocky overcloud host image source
+stackhpc_rocky_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/8/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-8.qcow2"
+stackhpc_rocky_overcloud_host_image_version: "latest"
+
+# CentOS overcloud host image source
+stackhpc_centos_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/centos/stream-8/{{ stackhpc_centos_overcloud_host_image_version }}/overcloud-centos-stream-8.qcow2"
+stackhpc_centos_overcloud_host_image_version: "latest"
+
 ###############################################################################
 # Container image registry
 

From 28934c52a3e1735c2124d01ec458c15a9d608dbd Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 17 Feb 2023 16:20:14 +0000
Subject: [PATCH 11/16] fix ubuntu image url typo, add pulp url with auth

---
 etc/kayobe/pulp.yml     | 11 ++++++++++-
 etc/kayobe/stackhpc.yml |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index ad475a56d..900b71e52 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -16,8 +16,14 @@ pulp_proxy_url: "{{ omit }}"
 ###############################################################################
 # StackHPC Pulp server
 
+# The scheme - http/https of the StackHPC Pulp service.
+stackhpc_release_pulp_scheme: "https://"
+
+# The domain of the StackHPC Pulp service.
+stackhpc_release_pulp_domain: "ark.stackhpc.com"
+
 # Base URL of the StackHPC Pulp service.
-stackhpc_release_pulp_url: "https://ark.stackhpc.com"
+stackhpc_release_pulp_url: "{{ stackhpc_release_pulp_scheme }}{{ stackhpc_release_pulp_domain }}"
 
 # Credentials used to access the StackHPC Ark pulp server.
 stackhpc_release_pulp_username:
@@ -26,6 +32,9 @@ stackhpc_release_pulp_password:
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
+# Content URL of the StackHPC Pulp service, with basic auth.
+stackhpc_release_pulp_content_url_with_auth: "{{ stackhpc_release_pulp_scheme }}{{ stackhpc_release_pulp_username }}:{{ stackhpc_release_pulp_password }}@{{ stackhpc_release_pulp_domain }}/pulp/content"
+
 ###############################################################################
 # Debs
 
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 52b402adc..e0bbd2a67 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -136,7 +136,7 @@ stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
 stackhpc_download_overcloud_images: false
 
 # Ubuntu overcloud host image source
-stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image }}/overcloud-ubuntu-focal.qcow2"
+stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-focal.qcow2"
 stackhpc_ubuntu_overcloud_host_image_version: "latest"
 
 # Rocky overcloud host image source

From 69c25886a160f764f0427515c2578f557e222021 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 20 Feb 2023 15:40:20 +0000
Subject: [PATCH 12/16] move image sources to separate file & small fixes

---
 etc/kayobe/ansible/pulp-upload-image.yml      | 12 +++---------
 etc/kayobe/kolla/config/bifrost/bifrost.yml   |  2 +-
 etc/kayobe/stackhpc-overcloud-host-images.yml | 17 +++++++++++++++++
 etc/kayobe/stackhpc.yml                       | 17 -----------------
 4 files changed, 21 insertions(+), 27 deletions(-)
 create mode 100644 etc/kayobe/stackhpc-overcloud-host-images.yml

diff --git a/etc/kayobe/ansible/pulp-upload-image.yml b/etc/kayobe/ansible/pulp-upload-image.yml
index 41c192de0..b40513f00 100644
--- a/etc/kayobe/ansible/pulp-upload-image.yml
+++ b/etc/kayobe/ansible/pulp-upload-image.yml
@@ -54,12 +54,7 @@
         present_content:
           - relative_path: "{{ found_files.files[0].path | basename }}"
             sha256: "{{ file_stats.stat.checksum }}"
-      register: repo_details
 
-    # Squeezer wants the version as an int. The previous play only returns the
-    # href of the latest version.
-    # The version can be returned by querying that href, however the last
-    # number in the href is the version. regex_search is used to extract it
     - name: Create a new publication to point to this version
       pulp.squeezer.file_publication:
         pulp_url: "{{ remote_pulp_url }}"
@@ -67,7 +62,6 @@
         password: "{{ remote_pulp_password }}"
         repository: "{{ repository_name }}"
         state: present
-        version: "{{ repo_details.repository_version | regex_search('(\\d+)(?!.*\\d)') }}"
       register: publication_details
 
     - name: Update distribution for latest version
@@ -92,7 +86,7 @@
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
-      when: latest_distribution_details.changed is true
+      when: latest_distribution_details.changed
 
     - name: Update new images file with versioned path
       lineinfile:
@@ -100,11 +94,11 @@
         line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         {{ ansible_facts.date_time.iso8601_basic_short }}/{{ found_files.files[0].path | basename }}"
         create: true
-      when: latest_distribution_details.changed is true
+      when: latest_distribution_details.changed
 
     - name: Update new images file with latest path
       lineinfile:
         path: /tmp/updated_images.txt
         line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
-      when: latest_distribution_details.changed is true
+      when: latest_distribution_details.changed
diff --git a/etc/kayobe/kolla/config/bifrost/bifrost.yml b/etc/kayobe/kolla/config/bifrost/bifrost.yml
index cb8ddbf3e..f864fdd56 100644
--- a/etc/kayobe/kolla/config/bifrost/bifrost.yml
+++ b/etc/kayobe/kolla/config/bifrost/bifrost.yml
@@ -1,6 +1,6 @@
 ---
 # Use images from Ark.
-{% if stackhpc_download_overcloud_images %}
+{% if stackhpc_download_overcloud_images | bool %}
 use_cirros: true
 {% if os_distribution == 'ubuntu' %}
 cirros_deploy_image_upstream_url: "{{ stackhpc_ubuntu_overcloud_host_image_url }}"
diff --git a/etc/kayobe/stackhpc-overcloud-host-images.yml b/etc/kayobe/stackhpc-overcloud-host-images.yml
new file mode 100644
index 000000000..264e27f7d
--- /dev/null
+++ b/etc/kayobe/stackhpc-overcloud-host-images.yml
@@ -0,0 +1,17 @@
+---
+##############################
+# Overcloud host image sources
+
+stackhpc_download_overcloud_images: false
+
+# Ubuntu overcloud host image source
+stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-focal.qcow2"
+stackhpc_ubuntu_overcloud_host_image_version: "20221216T112110" # Not verified
+
+# Rocky overcloud host image source
+stackhpc_rocky_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/8/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-linux-8.qcow2"
+stackhpc_rocky_overcloud_host_image_version: "20221216T110126" # Not verified
+
+# CentOS overcloud host image source
+stackhpc_centos_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/centos/stream-8/{{ stackhpc_centos_overcloud_host_image_version }}/overcloud-centos-stream-8.qcow2"
+stackhpc_centos_overcloud_host_image_version: "20221216T103526" # Not verified
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index e0bbd2a67..38a3efded 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -130,23 +130,6 @@ stackhpc_kolla_source_version: "stackhpc/{{ openstack_release }}"
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
 stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
 
-###############################################################################
-# Overcloud host image source
-
-stackhpc_download_overcloud_images: false
-
-# Ubuntu overcloud host image source
-stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-focal.qcow2"
-stackhpc_ubuntu_overcloud_host_image_version: "latest"
-
-# Rocky overcloud host image source
-stackhpc_rocky_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/8/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-8.qcow2"
-stackhpc_rocky_overcloud_host_image_version: "latest"
-
-# CentOS overcloud host image source
-stackhpc_centos_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/centos/stream-8/{{ stackhpc_centos_overcloud_host_image_version }}/overcloud-centos-stream-8.qcow2"
-stackhpc_centos_overcloud_host_image_version: "latest"
-
 ###############################################################################
 # Container image registry
 

From ddd740ae42467000dafab50d572a0f0aec20421e Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 24 Mar 2023 17:29:32 +0000
Subject: [PATCH 13/16] Add image promotion workflow

---
 .../overcloud-host-image-promote.yml          | 121 ++++++++++++++++++
 etc/kayobe/ansible/pulp-image-promote.yml     |  40 ++++++
 ...upload-image.yml => pulp-image-upload.yml} |   5 +
 3 files changed, 166 insertions(+)
 create mode 100644 .github/workflows/overcloud-host-image-promote.yml
 create mode 100644 etc/kayobe/ansible/pulp-image-promote.yml
 rename etc/kayobe/ansible/{pulp-upload-image.yml => pulp-image-upload.yml} (96%)

diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml
new file mode 100644
index 000000000..81e20dd25
--- /dev/null
+++ b/.github/workflows/overcloud-host-image-promote.yml
@@ -0,0 +1,121 @@
+---
+name: Promote overcloud host image
+on:
+  workflow_dispatch:
+    inputs:
+      os_image:
+        description: Image to promote
+        type: choice
+        required: true
+        default: 'CentOS Stream 8'
+        options:
+          - 'CentOS Stream 8'
+          - 'Rocky Linux 8'
+          - 'Ubuntu Focal 20.04'
+      image_tag:
+        description: Tag to promote
+        type: string
+        required: true
+env:
+  ANSIBLE_FORCE_COLOR: True
+jobs:
+  overcloud-host-image-promote:
+    name: Promote overcloud host image
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: src/kayobe-config
+
+      - name: Clone StackHPC Kayobe repository
+        uses: actions/checkout@v3
+        with:
+          repository: stackhpc/kayobe
+          ref: refs/heads/stackhpc/xena
+          path: src/kayobe
+
+      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
+      - name: Install dependencies
+        run: |
+          sudo dnf -y install python3-virtualenv
+
+      - name: Setup networking
+        run: |
+          if ! ip l show breth1 >/dev/null 2>&1; then
+              sudo ip l add breth1 type bridge
+          fi
+          sudo ip l set breth1 up
+          if ! ip a show breth1 | grep 192.168.33.3/24; then
+              sudo ip a add 192.168.33.3/24 dev breth1
+          fi
+          if ! ip l show dummy1 >/dev/null 2>&1; then
+              sudo ip l add dummy1 type dummy
+          fi
+          sudo ip l set dummy1 up
+          sudo ip l set dummy1 master breth1
+
+      - name: Install Kayobe
+        run: |
+          mkdir -p venvs &&
+          pushd venvs &&
+          python3 -m venv kayobe &&
+          source kayobe/bin/activate &&
+          pip install -U pip &&
+          pip install ../src/kayobe
+
+      - name: Bootstrap the control host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe control host bootstrap
+
+      - name: Configure the seed host
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host configure
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
+      - name: Promote CentOS Stream 8 overcloud host image artifact
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
+          -e os_distribution='centos' \
+          -e os_release='8-stream'
+        env:
+          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: os_image == 'CentOS Stream 8'
+
+      - name: Promote Rocky Linux 8 overcloud host image artifact
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
+          -e os_distribution='rocky' \
+          -e os_release='8'
+        env:
+          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: os_image == 'Rocky Linux 8'
+
+      - name: Promote Ubuntu Focal 20.04 overcloud host image artifact
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
+          -e os_distribution='ubuntu' \
+          -e os_release='focal'
+        env:
+          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: os_image == 'Ubuntu Focal 20.04'
diff --git a/etc/kayobe/ansible/pulp-image-promote.yml b/etc/kayobe/ansible/pulp-image-promote.yml
new file mode 100644
index 000000000..d93d71d51
--- /dev/null
+++ b/etc/kayobe/ansible/pulp-image-promote.yml
@@ -0,0 +1,40 @@
+---
+# Tag is the version of the image to promote, stored in an env variable named OVERCLOUD_HOST_IMAGE_TAG
+- name: Promote an image to production
+  hosts: localhost
+  vars:
+    remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
+    remote_pulp_username: "{{ stackhpc_image_repository_username }}"
+    remote_pulp_password: "{{ stackhpc_image_repository_password }}"
+    repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
+    base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
+    promotion_tag: "{{ lookup('env', 'OVERCLOUD_HOST_IMAGE_TAG') }}"
+  tasks:
+
+    - name: Check whether the image exists
+      pulp.squeezer.file_distribution:
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        name: "{{ repository_name }}_{{ promotion_tag }}"
+        base_path: "{{ base_path }}/{{ promotion_tag }}"
+      register: distribution_details
+
+    - name: Fail if the image does not exist
+      fail:
+        msg: "Image {{ promotion_tag }} does not exist"
+      when: distribution_details.distribution is none
+
+    - name: Ensure production content guard is set
+      pulp.squeezer.file_distribution:
+        pulp_url: "{{ remote_pulp_url }}"
+        username: "{{ remote_pulp_username }}"
+        password: "{{ remote_pulp_password }}"
+        name: "{{ repository_name }}_{{ promotion_tag }}"
+        base_path: "{{ base_path }}/{{ promotion_tag }}"
+        content_guard: release
+        state: present
+
+    - name: Print version tag and os
+      debug:
+        msg: "Promoted tag: {{ promotion_tag }}"
diff --git a/etc/kayobe/ansible/pulp-upload-image.yml b/etc/kayobe/ansible/pulp-image-upload.yml
similarity index 96%
rename from etc/kayobe/ansible/pulp-upload-image.yml
rename to etc/kayobe/ansible/pulp-image-upload.yml
index b40513f00..b2945c83f 100644
--- a/etc/kayobe/ansible/pulp-upload-image.yml
+++ b/etc/kayobe/ansible/pulp-image-upload.yml
@@ -102,3 +102,8 @@
         line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
+
+    - name: Print version tag
+      debug:
+        msg: "New tag: {{ ansible_facts.date_time.iso8601_basic_short }}"
+      when: latest_distribution_details.changed

From bba4be6ff6945af1418c1656fcb9d51732cccb5a Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 9 May 2023 16:46:29 +0100
Subject: [PATCH 14/16] Rename pulp overcloud host image playbooks

Rename ansible playbooks handling overcloud host image promotion
---
 .github/workflows/overcloud-host-image-build.yml            | 6 +++---
 .github/workflows/overcloud-host-image-promote.yml          | 6 +++---
 .../{pulp-image-promote.yml => pulp-host-image-promote.yml} | 0
 .../{pulp-image-upload.yml => pulp-host-image-upload.yml}   | 0
 4 files changed, 6 insertions(+), 6 deletions(-)
 rename etc/kayobe/ansible/{pulp-image-promote.yml => pulp-host-image-promote.yml} (100%)
 rename etc/kayobe/ansible/{pulp-image-upload.yml => pulp-host-image-upload.yml} (100%)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 9518428e4..cf9b62806 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -104,7 +104,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
           -e os_distribution='centos'
         env:
@@ -127,7 +127,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
           -e os_distribution='rocky'
         env:
@@ -156,7 +156,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-upload-image.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
           -e os_distribution='ubuntu'
         env:
diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml
index 81e20dd25..f43d783a6 100644
--- a/.github/workflows/overcloud-host-image-promote.yml
+++ b/.github/workflows/overcloud-host-image-promote.yml
@@ -83,7 +83,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \
           -e image_path='/opt/kayobe/images/overcloud-centos-stream-8' \
           -e os_distribution='centos' \
           -e os_release='8-stream'
@@ -97,7 +97,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \
           -e image_path='/opt/kayobe/images/overcloud-rocky-linux-8' \
           -e os_distribution='rocky' \
           -e os_release='8'
@@ -111,7 +111,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run \
-          src/kayobe-config/etc/kayobe/ansible/pulp-image-promote.yml \
+          src/kayobe-config/etc/kayobe/ansible/pulp-host-image-promote.yml \
           -e image_path='/opt/kayobe/images/overcloud-ubuntu-focal' \
           -e os_distribution='ubuntu' \
           -e os_release='focal'
diff --git a/etc/kayobe/ansible/pulp-image-promote.yml b/etc/kayobe/ansible/pulp-host-image-promote.yml
similarity index 100%
rename from etc/kayobe/ansible/pulp-image-promote.yml
rename to etc/kayobe/ansible/pulp-host-image-promote.yml
diff --git a/etc/kayobe/ansible/pulp-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml
similarity index 100%
rename from etc/kayobe/ansible/pulp-image-upload.yml
rename to etc/kayobe/ansible/pulp-host-image-upload.yml

From 988d93010f9b64f4f2848aa11d31f0db38cd9bde Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 9 May 2023 16:56:18 +0100
Subject: [PATCH 15/16] Revert changes to pull pulp overcloud host images

---
 .../environments/ci-builder/stackhpc-ci.yml     |  3 ---
 etc/kayobe/kolla/config/bifrost/bifrost.yml     | 12 ------------
 etc/kayobe/pulp.yml                             | 11 +----------
 etc/kayobe/stackhpc-overcloud-host-images.yml   | 17 -----------------
 4 files changed, 1 insertion(+), 42 deletions(-)
 delete mode 100644 etc/kayobe/kolla/config/bifrost/bifrost.yml
 delete mode 100644 etc/kayobe/stackhpc-overcloud-host-images.yml

diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 13359f72f..4a9e50f1c 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -82,6 +82,3 @@ stackhpc_docker_registry_password: !vault |
   34386133383366326639353432386235336132663839333337323739633434613934346462363031
   3265323831663964360a643962346231386462323236373963633066393736323234303833363535
   3664
-
-stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
-stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"
diff --git a/etc/kayobe/kolla/config/bifrost/bifrost.yml b/etc/kayobe/kolla/config/bifrost/bifrost.yml
deleted file mode 100644
index f864fdd56..000000000
--- a/etc/kayobe/kolla/config/bifrost/bifrost.yml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-# Use images from Ark.
-{% if stackhpc_download_overcloud_images | bool %}
-use_cirros: true
-{% if os_distribution == 'ubuntu' %}
-cirros_deploy_image_upstream_url: "{{ stackhpc_ubuntu_overcloud_host_image_url }}"
-{% elif os_distribution == 'rocky' %}
-cirros_deploy_image_upstream_url: "{{ stackhpc_rocky_overcloud_host_image_url }}"
-{% else %}
-cirros_deploy_image_upstream_url: "{{ stackhpc_centos_overcloud_host_image_url }}"
-{% endif %}
-{% endif %}
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 900b71e52..ad475a56d 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -16,14 +16,8 @@ pulp_proxy_url: "{{ omit }}"
 ###############################################################################
 # StackHPC Pulp server
 
-# The scheme - http/https of the StackHPC Pulp service.
-stackhpc_release_pulp_scheme: "https://"
-
-# The domain of the StackHPC Pulp service.
-stackhpc_release_pulp_domain: "ark.stackhpc.com"
-
 # Base URL of the StackHPC Pulp service.
-stackhpc_release_pulp_url: "{{ stackhpc_release_pulp_scheme }}{{ stackhpc_release_pulp_domain }}"
+stackhpc_release_pulp_url: "https://ark.stackhpc.com"
 
 # Credentials used to access the StackHPC Ark pulp server.
 stackhpc_release_pulp_username:
@@ -32,9 +26,6 @@ stackhpc_release_pulp_password:
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
-# Content URL of the StackHPC Pulp service, with basic auth.
-stackhpc_release_pulp_content_url_with_auth: "{{ stackhpc_release_pulp_scheme }}{{ stackhpc_release_pulp_username }}:{{ stackhpc_release_pulp_password }}@{{ stackhpc_release_pulp_domain }}/pulp/content"
-
 ###############################################################################
 # Debs
 
diff --git a/etc/kayobe/stackhpc-overcloud-host-images.yml b/etc/kayobe/stackhpc-overcloud-host-images.yml
deleted file mode 100644
index 264e27f7d..000000000
--- a/etc/kayobe/stackhpc-overcloud-host-images.yml
+++ /dev/null
@@ -1,17 +0,0 @@
----
-##############################
-# Overcloud host image sources
-
-stackhpc_download_overcloud_images: false
-
-# Ubuntu overcloud host image source
-stackhpc_ubuntu_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/ubuntu/focal/{{ stackhpc_ubuntu_overcloud_host_image_version }}/overcloud-ubuntu-focal.qcow2"
-stackhpc_ubuntu_overcloud_host_image_version: "20221216T112110" # Not verified
-
-# Rocky overcloud host image source
-stackhpc_rocky_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/rocky/8/{{ stackhpc_rocky_overcloud_host_image_version }}/overcloud-rocky-linux-8.qcow2"
-stackhpc_rocky_overcloud_host_image_version: "20221216T110126" # Not verified
-
-# CentOS overcloud host image source
-stackhpc_centos_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/{{ openstack_release }}/centos/stream-8/{{ stackhpc_centos_overcloud_host_image_version }}/overcloud-centos-stream-8.qcow2"
-stackhpc_centos_overcloud_host_image_version: "20221216T103526" # Not verified

From 6c9958eb729253566d51bea88babb5abe837c785 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com>
Date: Wed, 10 May 2023 11:15:21 +0100
Subject: [PATCH 16/16] Correct host image promote workflow input syntax

Co-authored-by: Mark Goddard <mark@stackhpc.com>
---
 .github/workflows/overcloud-host-image-promote.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml
index f43d783a6..cacbd575e 100644
--- a/.github/workflows/overcloud-host-image-promote.yml
+++ b/.github/workflows/overcloud-host-image-promote.yml
@@ -88,7 +88,7 @@ jobs:
           -e os_distribution='centos' \
           -e os_release='8-stream'
         env:
-          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: os_image == 'CentOS Stream 8'
 
@@ -102,7 +102,7 @@ jobs:
           -e os_distribution='rocky' \
           -e os_release='8'
         env:
-          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: os_image == 'Rocky Linux 8'
 
@@ -116,6 +116,6 @@ jobs:
           -e os_distribution='ubuntu' \
           -e os_release='focal'
         env:
-          OVERCLOUD_HOST_IMAGE_TAG: image_tag
+          OVERCLOUD_HOST_IMAGE_TAG: ${{ inputs.image_tag }}
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: os_image == 'Ubuntu Focal 20.04'