From 4c67cf34a1e2f1c3da9b0682b40ecd2d3db32079 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 12 Jan 2024 12:06:41 +0000
Subject: [PATCH 01/25] CI: Add upgrade tests

Adds two upgrade test jobs: Ubuntu Jammy OVS and Rocky 9 OVN.

These jobs reuse the existing aio reusable workflow, which has a new
'upgrade' input parameter. When testing an upgrade, the previous release
is first deployed, then the aio is upgraded to the current release
before running RefStack tests.
---
 .github/workflows/stackhpc-all-in-one.yml   | 96 +++++++++++++++++++--
 .github/workflows/stackhpc-pull-request.yml | 38 ++++++++
 2 files changed, 126 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 8b67fec66..4f759a2b5 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -55,6 +55,10 @@ on:
         description: Whether to run the workflow (workaround for required status checks issue)
         type: boolean
         default: true
+      upgrade:
+        description: Whether to perform an upgrade
+        type: boolean
+        default: false
     secrets:
       KAYOBE_VAULT_PASSWORD:
         required: true
@@ -76,10 +80,27 @@ jobs:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
+      # NOTE(upgrade): Reference the PREVIOUS release here.
+      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-yoga
+      # NOTE(upgrade): Reference the PREVIOUS release branch here.
+      PREVIOUS_BRANCH: stackhpc/yoga
     steps:
-      - uses: actions/checkout@v3
+      # If testing upgrade, checkout previous release, otherwise checkout current branch
+      - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
+        uses: actions/checkout@v3
         with:
-           submodules: true
+          ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }}
+          submodules: true
+
+      - name: Output Kayobe image
+        id: kayobe_image
+        run: |
+          if ${{ inputs.upgrade }}; then
+            kayobe_image=$PREVIOUS_KAYOBE_IMAGE
+          else
+            kayobe_image=$KAYOBE_IMAGE
+          fi
+          echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT
 
       - name: Output image tag
         id: image_tag
@@ -125,6 +146,7 @@ jobs:
           aio_vm_flavor = "${{ env.VM_FLAVOR }}"
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
           EOF
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
@@ -135,6 +157,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
 
       - name: Terraform Plan
         run: terraform plan
@@ -206,8 +229,14 @@ jobs:
           cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
+      # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
+      - name: Pull previous Kayobe image
+        run: |
+          sudo docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
+        if: inputs.upgrade
+
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
-      - name: Pull latest Kayobe image
+      - name: Pull current Kayobe image
         run: |
           sudo docker image pull $KAYOBE_IMAGE
 
@@ -216,7 +245,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -226,7 +255,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -236,7 +265,7 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
@@ -246,11 +275,62 @@ jobs:
           sudo -E docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
-            $KAYOBE_IMAGE \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      # If testing upgrade, checkout the current release branch
+      # Stash changes to tracked files, and set clean=false to avoid removing untracked files.
+      - name: Stash config changes
+        run: git stash
+        if: inputs.upgrade
+
+      - name: Checkout current release config
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          clean: false
+        if: inputs.upgrade
+
+      - name: Pop stashed config changes
+        run: git stash pop
+        if: inputs.upgrade
+
+      # Now begin upgrade
+      - name: Host upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
+      - name: Host configure
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
+      - name: Service upgrade
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
@@ -266,7 +346,7 @@ jobs:
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
+          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' }}
           path: tempest-artifacts/*
 
       - name: Fail if any Tempest tests failed
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index c8aad3652..2a85a206f 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -143,3 +143,41 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
+
+  all-in-one-upgrade-ubuntu-jammy-ovs:
+    name: aio upgrade (Ubuntu Jammy OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovs
+      OS_CLOUD: sms-lab-release
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-upgrade-rocky-9-ovn:
+    name: aio upgrade (Rocky 9 OVN)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovn
+      OS_CLOUD: sms-lab-release
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

From 653c886c75ac0bb97bc71f043544ab01a9e0160d Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 14 Sep 2023 11:16:51 +0100
Subject: [PATCH 02/25] Move Kolla image tags to a separate file

This will make it easier to automate various tasks.
---
 etc/kayobe/kolla-image-tags.yml | 23 +++++++++++++++++++++++
 etc/kayobe/kolla/globals.yml    | 25 ++++++-------------------
 2 files changed, 29 insertions(+), 19 deletions(-)
 create mode 100644 etc/kayobe/kolla-image-tags.yml

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
new file mode 100644
index 000000000..d6416b169
--- /dev/null
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -0,0 +1,23 @@
+---
+# Dict of Kolla image tags to deploy for each service.
+# Each key is the tag variable prefix name, and the value is another dict,
+# where the key is the OS distro and the value is the tag to deploy.
+kolla_image_tags:
+  openstack:
+    rocky: zed-rocky-9-20230921T153510
+    ubuntu: zed-ubuntu-jammy-20230921T153510
+  bifrost:
+    rocky: zed-rocky-9-20230927T142529
+    ubuntu: zed-ubuntu-jammy-20230927T154443
+  ovn:
+    rocky: zed-rocky-9-20230925T132313
+    ubuntu: zed-ubuntu-jammy-20230821T155947
+  cloudkitty:
+    rocky: zed-rocky-9-20231114T124701
+    ubuntu: zed-ubuntu-jammy-20231114T124701
+  neutron:
+    rocky: zed-rocky-9-20231115T094053
+    ubuntu: zed-ubuntu-jammy-20231115T094053
+  opensearch:
+    rocky: zed-rocky-9-20231214T095452
+    ubuntu: zed-ubuntu-jammy-20231214T095452
diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index d2e20b974..911755ce1 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -10,25 +10,12 @@ enable_docker_repo: "{% raw %}{{ 'overcloud' not in group_names }}{% endraw %}"
 # This is necessary for os migrations where mixed clouds might be deployed
 kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}"
 
-kayobe_image_tags:
-  openstack:
-    rocky: zed-rocky-9-20230921T153510
-    ubuntu: zed-ubuntu-jammy-20230921T153510
-  bifrost:
-    rocky: zed-rocky-9-20230927T142529
-    ubuntu: zed-ubuntu-jammy-20230927T154443
-  ovn:
-    rocky: zed-rocky-9-20230925T132313
-    ubuntu: zed-ubuntu-jammy-20230821T155947
-  cloudkitty:
-    rocky: zed-rocky-9-20231114T124701
-    ubuntu: zed-ubuntu-jammy-20231114T124701
-  neutron:
-    rocky: zed-rocky-9-20231115T094053
-    ubuntu: zed-ubuntu-jammy-20231115T094053
-  opensearch:
-    rocky: zed-rocky-9-20231214T095452
-    ubuntu: zed-ubuntu-jammy-20231214T095452
+# Dict of Kolla image tags to deploy for each service.
+# Each key is the tag variable prefix name, and the value is another dict,
+# where the key is the OS distro and the value is the tag to deploy.
+# NOTE: This is defined in etc/kayobe/kolla-image-tags.yml.
+kolla_image_tags:
+{{ kolla_image_tags | to_nice_yaml | indent(width=4, first=true) }}
 
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"

From 0cfa5489eafd5e56192461a2b51ab299dba412ce Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 3 Oct 2023 11:07:32 +0100
Subject: [PATCH 03/25] Add base distro version to Kolla image tag definitions

---
 etc/kayobe/kolla-image-tags.yml | 24 ++++++++++++------------
 etc/kayobe/pulp.yml             |  5 +++--
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index d6416b169..c7b669fb5 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -4,20 +4,20 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky: zed-rocky-9-20230921T153510
-    ubuntu: zed-ubuntu-jammy-20230921T153510
+    rocky-9: zed-rocky-9-20230921T153510
+    ubuntu-jammy: zed-ubuntu-jammy-20230921T153510
   bifrost:
-    rocky: zed-rocky-9-20230927T142529
-    ubuntu: zed-ubuntu-jammy-20230927T154443
+    rocky-9: zed-rocky-9-20230927T142529
+    ubuntu-jammy: zed-ubuntu-jammy-20230927T154443
   ovn:
-    rocky: zed-rocky-9-20230925T132313
-    ubuntu: zed-ubuntu-jammy-20230821T155947
+    rocky-9: zed-rocky-9-20230925T132313
+    ubuntu-jammy: zed-ubuntu-jammy-20230821T155947
   cloudkitty:
-    rocky: zed-rocky-9-20231114T124701
-    ubuntu: zed-ubuntu-jammy-20231114T124701
+    rocky-9: zed-rocky-9-20231114T124701
+    ubuntu-jammy: zed-ubuntu-jammy-20231114T124701
   neutron:
-    rocky: zed-rocky-9-20231115T094053
-    ubuntu: zed-ubuntu-jammy-20231115T094053
+    rocky-9: zed-rocky-9-20231115T094053
+    ubuntu-jammy: zed-ubuntu-jammy-20231115T094053
   opensearch:
-    rocky: zed-rocky-9-20231214T095452
-    ubuntu: zed-ubuntu-jammy-20231214T095452
+    rocky-9: zed-rocky-9-20231214T095452
+    ubuntu-jammy: zed-ubuntu-jammy-20231214T095452
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index d305f1d91..9207f5239 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -522,8 +522,9 @@ stackhpc_pulp_images_kolla:
 
 # List of images for each base distribution which should not/cannot be built.
 stackhpc_kolla_unbuildable_images:
-  ubuntu: []
-  rocky: []
+  ubuntu-jammy: []
+  rocky-9:
+    - iscsid
 
 # Whitespace-separated list of regular expressions matching Kolla image names.
 # Usage is similar to kolla-build CLI arguments.

From fafc68ae0ced87124cbf3290fa90456a3b5a27e4 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 3 Oct 2023 11:08:05 +0100
Subject: [PATCH 04/25] Add kolla-images.py

---
 tools/kolla-images.py | 365 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 365 insertions(+)
 create mode 100755 tools/kolla-images.py

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
new file mode 100755
index 000000000..f30b989b5
--- /dev/null
+++ b/tools/kolla-images.py
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+
+"""
+Script to manage Kolla container image tags.
+
+Background:
+In Kolla Ansible each container is deployed using a specific image.
+Typically the image is named the same as the container, with underscores
+replaced by dashes, however there are some exceptions. Sometimes multiple
+containers use the same image.
+
+The image tag deployed by each container is defined by a Kolla Ansible variable
+named <container>_tag. There are also intermediate tag variables to make it
+easier to set the tag for all containers in a service, e.g. nova_tag is the
+default for nova_api_tag, nova_compute, etc. There is a global default tag
+defined by openstack_tag. This setup forms a hierarchy of tag variables.
+
+This script captures this logic, as well as exceptions to these rules.
+"""
+
+import argparse
+import inspect
+import json
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from typing import Dict, List, Optional
+
+import yaml
+
+
+# Dict of Kolla image tags to deploy for each service.
+# Each key is the tag variable prefix name, and the value is another dict,
+# where the key is the OS distro and the value is the tag to deploy.
+# This is the content of etc/kayobe/kolla-image-tags.yml.
+KollaImageTags = Dict[str, Dict[str, str]]
+
+# Maps a Kolla image to a list of containers that use the image.
+IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = {
+    "haproxy": [
+        "glance_tls_proxy",
+        "neutron_tls_proxy",
+    ],
+    "mariadb-server": [
+        "mariadb",
+        "mariabackup",
+    ],
+    "neutron-eswitchd": [
+        "neutron_mlnx_agent",
+    ],
+    "neutron-metadata-agent": [
+        "neutron_metadata_agent",
+        "neutron_ovn_metadata_agent",
+    ],
+    "nova-conductor": [
+        "nova_super_conductor",
+        "nova_conductor",
+    ],
+    "prometheus-v2-server": [
+        "prometheus_server",
+    ],
+}
+
+# Maps a container to the parent tag variable in the hierarchy.
+CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: Dict[str, str] = {
+    "cron": "common",
+    "fluentd": "common",
+    "glance_tls_proxy": "haproxy",
+    "hacluster_corosync": "openstack",
+    "hacluster_pacemaker": "openstack",
+    "hacluster_pacemaker_remote": "openstack",
+    "heat_api_cfn": "heat",
+    "ironic_neutron_agent": "neutron",
+    "kolla_toolbox": "common",
+    "mariabackup": "mariadb",
+    "neutron_eswitchd": "neutron_mlnx_agent",
+    "neutron_tls_proxy": "haproxy",
+    "nova_compute_ironic": "nova",
+    "redis_sentinel": "openstack",
+    "swift_object_expirer": "swift",
+    "tgtd": "iscsi",
+}
+
+# List of supported base distributions and versions.
+SUPPORTED_BASE_DISTROS = [
+    "rocky-9",
+    "ubuntu-jammy",
+]
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible")
+    subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")
+
+    subparser = subparsers.add_parser("check-tags", help="Check specified tags for each image exist in the Ark registry")
+    subparser.add_argument("--registry", required=True, help="Hostname of container image registry")
+    subparser.add_argument("--namespace", required=True, help="Namespace in container image registry")
+
+    subparsers.add_parser("list-containers", help="List supported containers based on pulp.yml")
+
+    subparsers.add_parser("list-images", help="List supported images based on pulp.yml")
+
+    subparsers.add_parser("list-tags", help="List tags for each image based on kolla-image-tags.yml")
+
+    subparsers.add_parser("list-tag-vars", help="List Kolla Ansible tag variables")
+
+    return parser.parse_args()
+
+
+def get_abs_path(relative_path: str) -> str:
+    """Return the absolute path of a file in SKC."""
+    script_path = pathlib.Path(inspect.getfile(inspect.currentframe()))
+    return script_path.parent.parent / relative_path
+
+
+def read_images(images_file: str) -> List[str]:
+    """Read image list from pulp.yml config file."""
+    with open(get_abs_path(images_file), "r") as f:
+        variables = yaml.safe_load(f)
+    return variables["stackhpc_pulp_images_kolla"]
+
+
+def read_unbuildable_images(images_file: str) -> Dict[str, List[str]]:
+    """Read unbuildable image list from pulp.yml config file."""
+    with open(get_abs_path(images_file), "r") as f:
+        variables = yaml.safe_load(f)
+    return variables["stackhpc_kolla_unbuildable_images"]
+
+
+def read_kolla_image_tags(tags_file: str) -> KollaImageTags:
+    """Read kolla image tags kolla-image-tags.yml config file."""
+    with open(get_abs_path(tags_file), "r") as f:
+        variables = yaml.safe_load(f)
+    return variables["kolla_image_tags"]
+
+
+def get_containers(image):
+    """Return a list of containers that use the specified image."""
+    default_container = image.replace('-', '_')
+    return IMAGE_TO_CONTAINERS_EXCEPTIONS.get(image, [default_container])
+
+
+def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[str], container: str) -> str:
+    """Return the parent tag variable for a container in the tag variable hierarchy."""
+
+    if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS:
+        parent = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container]
+        if parent in kolla_image_tags:
+            return parent
+
+    def tag_key(tag):
+        """Return a sort key to order the tags."""
+        if tag == "openstack":
+            # This is the default tag.
+            return 0
+        elif tag != container and container.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]):
+            # Prefix match - sort by the longest match.
+            return -len(tag)
+        else:
+            # No match.
+            return 1
+
+    return sorted(kolla_image_tags.keys(), key=tag_key)[0]
+
+
+def get_parent_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str:
+    """Return the tag used by the parent in the hierarchy."""
+    parent_tag_name = get_parent_tag_name(kolla_image_tags, base_distro, container)
+    return kolla_image_tags[parent_tag_name][base_distro]
+
+
+def get_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str:
+    """Return the tag for a container."""
+    container_tag = kolla_image_tags.get(container, {}).get(base_distro)
+    if container_tag:
+        return container_tag
+
+    return get_parent_tag(kolla_image_tags, base_distro, container)
+
+
+def get_tags(base_distros: List[str], kolla_image_tags: KollaImageTags) -> Dict[str, List[str]]:
+    """Return a list of tags used for each image."""
+    images = read_images("etc/kayobe/pulp.yml")
+    unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml")
+    image_tags: Dict[str, List[str]] = {}
+    for base_distro in base_distros:
+        for image in images:
+            if image not in unbuildable_images[base_distro]:
+                for container in get_containers(image):
+                    tag = get_tag(kolla_image_tags, base_distro, container)
+                    tags = image_tags.setdefault(image, [])
+                    if tag not in tags:
+                        tags.append(tag)
+    return image_tags
+
+
+def get_openstack_release() -> str:
+    """Return the OpenStack release."""
+    with open(get_abs_path(".gitreview"), "r") as f:
+        gitreview = f.readlines()
+    for line in gitreview:
+        if "=" not in line:
+            continue
+        key, value = line.split("=")
+        if key.strip().strip() == "defaultbranch":
+            value = value.strip().rstrip()
+            prefix = "stable/"
+            assert value.startswith(prefix)
+            return value[len(prefix):]
+    raise Exception("Failed to determine OpenStack release")
+
+
+def validate(kolla_image_tags: KollaImageTags):
+    """Validate the kolla_image_tags variable."""
+    tag_var_re = re.compile(r"^[a-z0-9_]+$")
+    openstack_release = get_openstack_release()
+    tag_res = {
+        base_distro: re.compile(f"^{openstack_release}-{base_distro}-[\d]{{8}}T[\d]{{6}}$")
+        for base_distro in SUPPORTED_BASE_DISTROS
+    }
+    errors = []
+    if "openstack" not in kolla_image_tags:
+        errors.append("Missing default openstack tag")
+    for tag_var, base_distros in kolla_image_tags.items():
+        if not tag_var_re.match(tag_var):
+            errors.append(f"Key {tag_var} does not match expected pattern. It should match {tag_var_re.pattern}")
+        for base_distro, tag in base_distros.items():
+            if base_distro not in SUPPORTED_BASE_DISTROS:
+                errors.append(f"{tag_var}: base distro {base_distro} not supported. Options: {SUPPORTED_BASE_DISTROS}")
+                continue
+            if not tag_res[base_distro].match(tag):
+                errors.append(f"{tag_var}: {base_distro}: tag {tag} does not match expected pattern. It should match {tag_res[base_distro].pattern}")
+    if errors:
+        print("Errors in kolla_image_tags variable:")
+        for error in errors:
+            print(error)
+        sys.exit(1)
+
+
+def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, registry: str, namespace: str):
+    """Check whether expected tags are present in container image registry."""
+    image_tags = get_tags(base_distros, kolla_image_tags)
+
+    missing = {}
+    for image, tags in image_tags.items():
+        for _ in range(3):
+            try:
+                output = subprocess.check_output(f"skopeo list-tags docker://{registry}/{namespace}/{image}", shell=True)
+            except Exception as e:
+                exc = e
+            else:
+                break
+        else:
+            raise exc
+        ark_tags = json.loads(output)["Tags"]
+        missing_tags = set(tags) - set(ark_tags)
+        if missing_tags:
+            missing[image] = list(missing_tags)
+
+    if missing:
+        print(f"ERROR: Some expected tags not found in {namespace} namespace")
+        print(yaml.dump(missing, indent=2))
+        sys.exit(1)
+
+
+def check_hierarchy(kolla_ansible_path: str):
+    """Check the tag variable hierarchy against Kolla Ansible variables."""
+    cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml"""
+    hierarchy_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path))
+    hierarchy = yaml.safe_load(hierarchy_str)
+    # This one is not a container:
+    hierarchy.pop("octavia_amp_image_tag")
+    tag_var_re = re.compile(r"^([a-z0-9_]+)_tag$")
+    parent_re = re.compile(r"{{[\s]*([a-z0-9_]+)_tag[\s]*}}")
+    hierarchy = {
+        tag_var_re.match(tag_var).group(1): parent_re.match(parent).group(1)
+        for tag_var, parent in hierarchy.items()
+    }
+    kolla_image_tags: KollaImageTags = {image: {} for image in hierarchy}
+    kolla_image_tags["openstack"] = {}
+    errors = []
+    for tag_var, expected in hierarchy.items():
+        parent = get_parent_tag_name(kolla_image_tags, None, tag_var)
+        if parent != expected:
+            errors.append((tag_var, parent, expected))
+    if errors:
+        print("Errors:")
+    for tag_var, parent, expected in errors:
+        print(f"{tag_var} -> {parent} != {expected}")
+    if errors:
+        sys.exit(1)
+
+
+def list_containers(base_distros: List[str]):
+    """List supported containers."""
+    images = read_images("etc/kayobe/pulp.yml")
+    unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml")
+    containers = set()
+    for base_distro in base_distros:
+        for image in images:
+            if image not in unbuildable_images[base_distro]:
+                containers |= set(get_containers(image))
+    print(yaml.dump(sorted(containers)))
+
+
+def list_images(base_distros: List[str]):
+    """List supported images."""
+    images = read_images("etc/kayobe/pulp.yml")
+    print(yaml.dump(images))
+
+
+def list_tags(base_distros: List[str], kolla_image_tags: KollaImageTags):
+    """List tags used by each image."""
+    image_tags = get_tags(base_distros, kolla_image_tags)
+
+    print(yaml.dump(image_tags))
+
+
+def list_tag_vars(kolla_image_tags: KollaImageTags):
+    """List tag variables."""
+    tag_vars = []
+    for tag_var in kolla_image_tags:
+        if tag_var == "openstack":
+            default = ""
+        else:
+            parent_tag_name = get_parent_tag_name(kolla_image_tags, None, tag_var)
+            default = f" | default({parent_tag_name}_tag)"
+        tag_vars.append(f"{tag_var}_tag: \"{{{{ kolla_image_tags['{tag_var}'][kolla_base_distro_and_version]{default} }}}}\"")
+
+    for tag_var in tag_vars:
+        print(tag_var)
+
+
+def main():
+    args = parse_args()
+    kolla_image_tags = read_kolla_image_tags("etc/kayobe/kolla-image-tags.yml")
+    base_distros = args.base_distros.split(",")
+
+    validate(kolla_image_tags)
+
+    if args.command == "check-hierarchy":
+        check_hierarchy(args.kolla_ansible_path)
+    elif args.command == "check-tags":
+        check_tags(base_distros, kolla_image_tags, args.registry, args.namespace)
+    elif args.command == "list-containers":
+        list_containers(base_distros)
+    elif args.command == "list-images":
+        list_images(base_distros)
+    elif args.command == "list-tags":
+        list_tags(base_distros, kolla_image_tags)
+    elif args.command == "list-tag-vars":
+        list_tag_vars(kolla_image_tags)
+    else:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

From 8d8f6e0a8f330a8d0245b493412f6bf084125c8e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 23 Oct 2023 16:46:34 +0100
Subject: [PATCH 05/25] Remove iscsid from Pulp image list

We no longer require it since Ironic dropped the iscsi deploy driver.
---
 etc/kayobe/pulp.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 9207f5239..736423f28 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -523,8 +523,7 @@ stackhpc_pulp_images_kolla:
 # List of images for each base distribution which should not/cannot be built.
 stackhpc_kolla_unbuildable_images:
   ubuntu-jammy: []
-  rocky-9:
-    - iscsid
+  rocky-9: []
 
 # Whitespace-separated list of regular expressions matching Kolla image names.
 # Usage is similar to kolla-build CLI arguments.

From eee54814512d0ae187cd12924b2c0d641f4e331e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 24 Oct 2023 16:40:05 +0100
Subject: [PATCH 06/25] CI: Trigger container image promotion automatically
 after merge

No more forgotten image promotions!
---
 .github/workflows/stackhpc-promote.yml | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml
index e5c254381..bbd9279b5 100644
--- a/.github/workflows/stackhpc-promote.yml
+++ b/.github/workflows/stackhpc-promote.yml
@@ -1,5 +1,5 @@
 ---
-name: Promote package repositories
+name: Promote Pulp repositories
 on:
   push:
     branches:
@@ -7,7 +7,7 @@ on:
       - stackhpc/zed
 jobs:
   promote:
-    name: Trigger package repository promotion
+    name: Trigger Pulp promotion workflows
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-latest
     permissions: {}
@@ -27,3 +27,19 @@ jobs:
       - name: Display link to package repository promotion workflows
         run: |
           echo "::notice Package repository promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-promote.yml"
+
+      # NOTE(mgoddard): Trigger another CI workflow in the
+      # stackhpc-release-train repository.
+      - name: Trigger container image promotion
+        run: |
+          gh workflow run \
+          container-promote.yml \
+          --repo stackhpc/stackhpc-release-train \
+          --ref main \
+          -f kayobe_config_branch=${{ github.ref_name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+
+      - name: Display link to container image promotion workflows
+        run: |
+          echo "::notice Container image promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-promote.yml"

From d1262ba67eb402f2b1668df3811ed943a6efdae8 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 24 Oct 2023 16:42:13 +0100
Subject: [PATCH 07/25] Automatically define Kolla Ansible tag variables in
 globals.yml

Use the new kolla-images.py script to automatically define tag variables
in globals.yml based on the tags that are defined in kolla_image_tags.
---
 etc/kayobe/kolla/globals.yml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 911755ce1..b156f050a 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -10,6 +10,9 @@ enable_docker_repo: "{% raw %}{{ 'overcloud' not in group_names }}{% endraw %}"
 # This is necessary for os migrations where mixed clouds might be deployed
 kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}"
 
+# Convenience variable for base distro and version string.
+kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}"
+
 # Dict of Kolla image tags to deploy for each service.
 # Each key is the tag variable prefix name, and the value is another dict,
 # where the key is the OS distro and the value is the tag to deploy.
@@ -17,12 +20,11 @@ kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %
 kolla_image_tags:
 {{ kolla_image_tags | to_nice_yaml | indent(width=4, first=true) }}
 
-openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
-bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"
-ovn_tag: "{% raw %}{{ kayobe_image_tags['ovn'][kolla_base_distro] }}{% endraw %}"
-cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
-neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}"
-opensearch_tag: "{% raw %}{{ kayobe_image_tags['opensearch'][kolla_base_distro] }}{% endraw %}"
+# Variables defining which tag to use for each container's image.
+{{ lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tag-vars') }}
+
+#############################################################################
+# RabbitMQ
 
 om_enable_rabbitmq_high_availability: true
 

From ae21932916b41d1d4fc2c64d4fd7d08a1f6bcd18 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 25 Oct 2023 14:31:31 +0100
Subject: [PATCH 08/25] CI: Add a job to check whether all expected images are
 in Test Pulp

---
 .github/workflows/stackhpc-check-tags.yml     | 48 +++++++++++++++++++
 .github/workflows/stackhpc-pull-request.yml   | 10 ++++
 etc/kayobe/ansible/check-tags.yml             | 39 +++++++++++++++
 etc/kayobe/ansible/requirements.yml           |  2 +-
 .../environments/ci-aio/stackhpc-ci.yml       |  6 +++
 5 files changed, 104 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/stackhpc-check-tags.yml
 create mode 100644 etc/kayobe/ansible/check-tags.yml

diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml
new file mode 100644
index 000000000..1fdb01307
--- /dev/null
+++ b/.github/workflows/stackhpc-check-tags.yml
@@ -0,0 +1,48 @@
+---
+# This workflow queries the Test Pulp server to check that all image tags
+# specified in kolla_image_tags are present.
+
+name: Check container image tags
+on:
+  workflow_call:
+    inputs:
+      kayobe_image:
+        description: Kayobe container image
+        type: string
+        required: true
+    secrets:
+      KAYOBE_VAULT_PASSWORD:
+        required: true
+
+env:
+  ANSIBLE_FORCE_COLOR: True
+jobs:
+  check-tags:
+    name: Check container image tags
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    permissions: {}
+    env:
+      KAYOBE_ENVIRONMENT: ci-aio
+      KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+      KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
+      - name: Pull latest Kayobe image
+        run: |
+          sudo docker image pull $KAYOBE_IMAGE
+
+      - name: Check container image tags
+        run: |
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
+            '$KAYOBE_CONFIG_PATH/ansible/check-tags.yml'
+        #env:
+          #KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index c8aad3652..cb236281b 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -76,6 +76,16 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
+  check-tags:
+    name: Check container image tags
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-check-tags.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
   all-in-one-ubuntu-jammy-ovs:
     name: aio (Ubuntu Jammy OVS)
     needs:
diff --git a/etc/kayobe/ansible/check-tags.yml b/etc/kayobe/ansible/check-tags.yml
new file mode 100644
index 000000000..3f0d22a68
--- /dev/null
+++ b/etc/kayobe/ansible/check-tags.yml
@@ -0,0 +1,39 @@
+---
+# This playbook queries the Pulp server to check that all image tags specified
+# in kolla_image_tags are present.
+
+- name: Check whether tags exist in Pulp container registry
+  hosts: localhost
+  tasks:
+    - name: Query images and tags
+      command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py list-tags
+      register: kolla_images_result
+      changed_when: false
+
+    - name: Set a fact about images and tags
+      set_fact:
+        kolla_images: "{{ kolla_images_result.stdout | from_yaml }}"
+
+    - name: Set a fact about the Pulp URL
+      set_fact:
+        pulp_url: "{{ stackhpc_repo_mirror_url }}"
+
+    # Use state=read and allow_missing=false to check for missing tags in test pulp.
+    - import_role:
+        name: stackhpc.pulp.pulp_container_content
+      vars:
+        pulp_container_content: >-
+          {%- set contents = [] -%}
+          {%- for image, tags in kolla_images.items() -%}
+          {%- set repository = kolla_docker_namespace ~ "/" ~ image -%}
+          {%- set content = {
+            "allow_missing": False,
+            "repository": repository,
+            "state": "read",
+            "tags": tags,
+          } -%}
+          {%- set _ = contents.append(content) -%}
+          {%- endfor -%}
+          {{ contents }}
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index fd685f1dc..0bae52204 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -7,7 +7,7 @@ collections:
   - name: pulp.squeezer
     version: 0.0.13
   - name: stackhpc.pulp
-    version: 0.5.2
+    version: 0.5.4
   - name: stackhpc.hashicorp
     version: 2.4.0
   - name: stackhpc.kayobe_workflows
diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
index ec5a82578..f9f425f2b 100644
--- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
@@ -62,3 +62,9 @@ stackhpc_docker_registry_password: !vault |
   38333133393730633666613965653364316162353337313330346164303631313731646461363461
   3963323635373866630a633533376339363734626664333765313665623662613764363038383735
   38646138376438643533376161376634653439386230353365316239613430363338
+
+# Override Pulp credentials to allow querying container image tags in the
+# check-tags.yml custom playbook.
+pulp_url: "{{ stackhpc_repo_mirror_url }}"
+pulp_username: "{{ stackhpc_docker_registry_username }}"
+pulp_password: "{{ stackhpc_docker_registry_password }}"

From 82a194a9f6c8daad500712fea988b4a9a88ce272 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 29 Jan 2024 14:35:05 +0000
Subject: [PATCH 09/25] kolla-images.py: Remove unnecessary strips, support
 unmaintained branches

---
 tools/kolla-images.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index f30b989b5..f2c69e27e 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -209,11 +209,11 @@ def get_openstack_release() -> str:
         if "=" not in line:
             continue
         key, value = line.split("=")
-        if key.strip().strip() == "defaultbranch":
-            value = value.strip().rstrip()
-            prefix = "stable/"
-            assert value.startswith(prefix)
-            return value[len(prefix):]
+        if key.strip() == "defaultbranch":
+            value = value.strip()
+            for prefix in ("stable/", "unmaintained/"):
+                if value.startswith(prefix):
+                    return value[len(prefix):]
     raise Exception("Failed to determine OpenStack release")
 
 

From 4e908585e8549de324b02d9d81ff472f4559a790 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 29 Jan 2024 14:35:26 +0000
Subject: [PATCH 10/25] kolla-images.py: Check for existence of skopeo

---
 tools/kolla-images.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index f2c69e27e..836f86090 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -246,6 +246,11 @@ def validate(kolla_image_tags: KollaImageTags):
 
 def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, registry: str, namespace: str):
     """Check whether expected tags are present in container image registry."""
+    try:
+        subprocess.check_output("type skopeo", shell=True)
+    except subprocess.CalledProcessError:
+        print("Failed to find skopeo. Please install it.")
+        sys.exit(1)
     image_tags = get_tags(base_distros, kolla_image_tags)
 
     missing = {}

From 03371903c04cec806849f0f1c4ab3d8d0a4a045c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 29 Jan 2024 15:35:21 +0000
Subject: [PATCH 11/25] zed: Update bifrost image for Ubuntu Jammy

The previous tag was not in Ark, but this one is present.
---
 etc/kayobe/kolla-image-tags.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index c7b669fb5..024b247a6 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -8,7 +8,7 @@ kolla_image_tags:
     ubuntu-jammy: zed-ubuntu-jammy-20230921T153510
   bifrost:
     rocky-9: zed-rocky-9-20230927T142529
-    ubuntu-jammy: zed-ubuntu-jammy-20230927T154443
+    ubuntu-jammy: zed-ubuntu-jammy-20231101T132522
   ovn:
     rocky-9: zed-rocky-9-20230925T132313
     ubuntu-jammy: zed-ubuntu-jammy-20230821T155947

From 0bdc266d94052caa7c60bbdebe98eb4b7790cb2e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 29 Jan 2024 15:38:23 +0000
Subject: [PATCH 12/25] kolla-images.py: Fix container to image tag lookup
 corner case

The failing case was the haproxy image used by the neutron_tls_proxy
container, if a tag is defined for the 'neutron' prefix but not
'haproxy'.
---
 tools/kolla-images.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index 836f86090..06e67cb1c 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -151,16 +151,18 @@ def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[
     """Return the parent tag variable for a container in the tag variable hierarchy."""
 
     if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS:
-        parent = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container]
-        if parent in kolla_image_tags:
-            return parent
+        prefix_var = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container]
+        if prefix_var in kolla_image_tags:
+            return prefix_var
+    else:
+        prefix_var = container
 
     def tag_key(tag):
         """Return a sort key to order the tags."""
         if tag == "openstack":
             # This is the default tag.
             return 0
-        elif tag != container and container.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]):
+        elif tag != prefix_var and prefix_var.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]):
             # Prefix match - sort by the longest match.
             return -len(tag)
         else:

From 3d842a1e0063d7c5f41ac38a6c92d9471153c940 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 29 Jan 2024 15:41:41 +0000
Subject: [PATCH 13/25] CI: Update to checkout action v4 in aio job

Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com>
---
 .github/workflows/stackhpc-all-in-one.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 4f759a2b5..9b437dcfc 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -87,7 +87,7 @@ jobs:
     steps:
       # If testing upgrade, checkout previous release, otherwise checkout current branch
       - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }}
           submodules: true
@@ -287,7 +287,7 @@ jobs:
         if: inputs.upgrade
 
       - name: Checkout current release config
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
           clean: false

From 5466b082d6e40daa1759e4d16c0a170733572518 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 14:46:43 +0000
Subject: [PATCH 14/25] Fix variable name

---
 etc/kayobe/kolla-image-tags.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index cd7ae9789..cbd30ae81 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -2,7 +2,7 @@
 # Dict of Kolla image tags to deploy for each service.
 # Each key is the tag variable prefix name, and the value is another dict,
 # where the key is the OS distro and the value is the tag to deploy.
-kayobe_image_tags:
+kolla_image_tags:
   openstack:
     rocky: 2023.1-rocky-9-20231011T200357
     ubuntu: 2023.1-ubuntu-jammy-20231011T200357
@@ -30,4 +30,3 @@ kayobe_image_tags:
   opensearch:
     rocky: 2023.1-rocky-9-20231214T151917
     ubuntu: 2023.1-ubuntu-jammy-20231214T151917
-

From 62b9e2bb611ee694d6baeb47a8076a3c4976d2f3 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 18:49:46 +0000
Subject: [PATCH 15/25] Fix issue with pulp sync

```
TASK [stackhpc.pulp.pulp_repository : include_tasks] ************************************************************************************************************************
Friday 02 February 2024  18:54:16 +0100 (0:00:00.189)       0:00:00.189 *******
fatal: [localhost]: FAILED! =>
  msg: |-
    The conditional check 'pulp_repository_container_repos | length > 0' failed. The error was: error while evaluating conditional (pulp_repository_container_repos | length
> 0): {{ stackhpc_pulp_repository_container_repos }}: {{ (stackhpc_pulp_repository_container_repos_kolla +
        stackhpc_pulp_repository_container_repos_ceph +
        stackhpc_pulp_repository_container_repos_hashicorp) | selectattr('required') }}: {%- set repos = [] -%} {%- for image in stackhpc_pulp_images_kolla_filtered -%} {%-
if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro]-%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ image -%} {%- set repo = {"name": image_repo} -%}
 {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} {%- endif -%} {%- endfor -%} {{ repos }}: 'dict object' has no attribute
 'rocky'

    The error appears to be in '/home/stack/kayobe-env-zed/src/kayobe-config/etc/kayobe/ansible/collections/ansible_collections/stackhpc/pulp/roles/pulp_repository/tasks/mai
n.yml': line 2, column 3, but may
    be elsewhere in the file depending on the exact syntax problem.

    The offending line appears to be:

    ---
    - include_tasks: container.yml
      ^ here
```
---
 etc/kayobe/kolla.yml | 2 ++
 etc/kayobe/pulp.yml  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 64ad47586..db077fe09 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -74,6 +74,8 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}"
 # Default is kolla_base_distro_version_default_map[kolla_base_distro].
 #kolla_base_distro_version:
 
+kolla_base_distro_and_version: "{{ kolla_base_distro }}-{{ kolla_base_distro_version }}"
+
 # URL of docker registry to use for Kolla images. Default is not set, in which
 # case Quay.io will be used.
 #kolla_docker_registry:
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 736423f28..39bbfe875 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -548,7 +548,7 @@ stackhpc_pulp_repository_container_repos_kolla_common:
 stackhpc_pulp_repository_container_repos_kolla: >-
   {%- set repos = [] -%}
   {%- for image in stackhpc_pulp_images_kolla_filtered -%}
-  {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro]-%}
+  {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro_and_version]-%}
   {%- set image_repo = kolla_docker_namespace ~ "/" ~ image -%}
   {%- set repo = {"name": image_repo} -%}
   {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%}

From d2424a9967593634cbc1bbd738a89f03b3a597e8 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 18:33:27 +0000
Subject: [PATCH 16/25] Fix for haproxy-ssh

---
 tools/kolla-images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index 06e67cb1c..6946b17c7 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -221,7 +221,7 @@ def get_openstack_release() -> str:
 
 def validate(kolla_image_tags: KollaImageTags):
     """Validate the kolla_image_tags variable."""
-    tag_var_re = re.compile(r"^[a-z0-9_]+$")
+    tag_var_re = re.compile(r"^[a-z0-9_-]+$")
     openstack_release = get_openstack_release()
     tag_res = {
         base_distro: re.compile(f"^{openstack_release}-{base_distro}-[\d]{{8}}T[\d]{{6}}$")

From e7a23b718057f67031d34c6ca3142997ae5317ca Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 18:56:46 +0000
Subject: [PATCH 17/25] Adjust tags to container distro version

---
 etc/kayobe/kolla-image-tags.yml | 36 ++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index cbd30ae81..6411078b8 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -4,29 +4,29 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky: 2023.1-rocky-9-20231011T200357
-    ubuntu: 2023.1-ubuntu-jammy-20231011T200357
+    rocky-9: 2023.1-rocky-9-20231011T200357
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231011T200357
   bifrost:
-    rocky: 2023.1-rocky-9-20231228T140806
-    ubuntu: 2023.1-ubuntu-jammy-20231228T140806
+    rocky-9: 2023.1-rocky-9-20231228T140806
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231228T140806
   cloudkitty:
-    rocky: 2023.1-rocky-9-20231115T110235
-    ubuntu: 2023.1-ubuntu-jammy-20231115T110235
+    rocky-9: 2023.1-rocky-9-20231115T110235
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231115T110235
   haproxy-ssh:
-    rocky: 2023.1-rocky-9-20240104T071640
-    ubuntu: 2023.1-ubuntu-jammy-20240104T071640
+    rocky-9: 2023.1-rocky-9-20240104T071640
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
   letsencrypt:
-    rocky: 2023.1-rocky-9-20240104T071640
-    ubuntu: 2023.1-ubuntu-jammy-20240104T071640
+    rocky-9: 2023.1-rocky-9-20240104T071640
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
   neutron:
-    rocky: 2023.1-rocky-9-20231220T222020
-    ubuntu: 2023.1-ubuntu-jammy-20231220T222020
+    rocky-9: 2023.1-rocky-9-20231220T222020
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   nova:
-    rocky: 2023.1-rocky-9-20231220T222020
-    ubuntu: 2023.1-ubuntu-jammy-20231220T222020
+    rocky-9: 2023.1-rocky-9-20231220T222020
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   octavia:
-    rocky: 2023.1-rocky-9-20231220T222020
-    ubuntu: 2023.1-ubuntu-jammy-20231220T222020
+    rocky-9: 2023.1-rocky-9-20231220T222020
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   opensearch:
-    rocky: 2023.1-rocky-9-20231214T151917
-    ubuntu: 2023.1-ubuntu-jammy-20231214T151917
+    rocky-9: 2023.1-rocky-9-20231214T151917
+    ubuntu-jammy: 2023.1-ubuntu-jammy-20231214T151917

From 806f83848c7071b8d93d18301c6fd185453c0663 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 15:16:52 +0000
Subject: [PATCH 18/25] 2023.1: Update to Rocky 9.3 based kolla images

These have been rebuilt using Rocky 9.3 snapshots.

Builds:

- neutron: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/7757690528
- others: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/7754959464
---
 etc/kayobe/kolla-image-tags.yml                       | 11 ++---------
 .../notes/kolla-rocky9.3-images-a3ff235485093191.yaml |  4 ++++
 2 files changed, 6 insertions(+), 9 deletions(-)
 create mode 100644 releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index 6411078b8..fb61b113e 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -4,29 +4,22 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky-9: 2023.1-rocky-9-20231011T200357
+    rocky-9: 2023.1-rocky-9-20240202T105928
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231011T200357
   bifrost:
-    rocky-9: 2023.1-rocky-9-20231228T140806
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231228T140806
   cloudkitty:
-    rocky-9: 2023.1-rocky-9-20231115T110235
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231115T110235
   haproxy-ssh:
-    rocky-9: 2023.1-rocky-9-20240104T071640
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
   letsencrypt:
-    rocky-9: 2023.1-rocky-9-20240104T071640
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
   neutron:
-    rocky-9: 2023.1-rocky-9-20231220T222020
+    rocky-9: 2023.1-rocky-9-20240202T145927
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   nova:
-    rocky-9: 2023.1-rocky-9-20231220T222020
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   octavia:
-    rocky-9: 2023.1-rocky-9-20231220T222020
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020
   opensearch:
-    rocky-9: 2023.1-rocky-9-20231214T151917
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231214T151917
diff --git a/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml b/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml
new file mode 100644
index 000000000..0c5e9ddcc
--- /dev/null
+++ b/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Rocky images have been rebuilt and are now based on Rocky 9.3.

From 7e44433e54ea820a8742ea8d2878900249e2a7aa Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 23:00:53 +0000
Subject: [PATCH 19/25] Update other variable

---
 etc/kayobe/pulp.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 39bbfe875..f10f5ab65 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -565,7 +565,7 @@ stackhpc_pulp_distribution_container_kolla_common:
 stackhpc_pulp_distribution_container_kolla: >-
   {%- set distributions = [] -%}
   {%- for image in stackhpc_pulp_images_kolla_filtered -%}
-  {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro]-%}
+  {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro_and_version]-%}
   {%- set image_repo = kolla_docker_namespace ~ "/" ~ image -%}
   {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%}
   {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%}

From 46d126d79e78ec428ad432c555cf15df143fbdd8 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 23:02:46 +0000
Subject: [PATCH 20/25] Another fixup

```
TASK [Query images and tags] ***************************************************
Friday 02 February 2024  20:39:16 +0000 (0:00:01.453)       0:00:01.529 *******
fatal: [localhost]: FAILED! => changed=false
  cmd:
  - /stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py
  - list-tags
  delta: '0:00:00.324461'
  end: '2024-02-02 20:39:17.748038'
  msg: non-zero return code
  rc: 1
  start: '2024-02-02 20:39:17.423577'
  stderr: |-
    Traceback (most recent call last):
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 372, in <module>
        main()
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 364, in main
        list_tags(base_distros, kolla_image_tags)
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 328, in list_tags
        image_tags = get_tags(base_distros, kolla_image_tags)
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 199, in get_tags
        tag = get_tag(kolla_image_tags, base_distro, container)
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 187, in get_tag
        return get_parent_tag(kolla_image_tags, base_distro, container)
      File "/stack/kayobe-automation-env/src/kayobe-config/etc/kayobe/../../tools/kolla-images.py", line 178, in get_parent_tag
        return kolla_image_tags[parent_tag_name][base_distro]
    KeyError: 'rocky-9'
  stderr_lines: <omitted>
  stdout: ''
  stdout_lines: <omitted>
```
---
 .automation           | 2 +-
 tools/kolla-images.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.automation b/.automation
index b00f285be..9ba253a9a 160000
--- a/.automation
+++ b/.automation
@@ -1 +1 @@
-Subproject commit b00f285be240e34c643c4bd93a877e56587f71fa
+Subproject commit 9ba253a9adba51011ce74aa87e708015dafc75fc
diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index 6946b17c7..11e18094d 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -152,7 +152,7 @@ def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[
 
     if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS:
         prefix_var = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container]
-        if prefix_var in kolla_image_tags:
+        if prefix_var in kolla_image_tags and (base_distro is None or base_distro in kolla_image_tags[prefix_var]):
             return prefix_var
     else:
         prefix_var = container

From 793f876a6aca1b13ade021ad941d9265e0db29c8 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 23:07:05 +0000
Subject: [PATCH 21/25] Revert "Fix for haproxy-ssh"

This reverts commit d2424a9967593634cbc1bbd738a89f03b3a597e8.
---
 tools/kolla-images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/kolla-images.py b/tools/kolla-images.py
index 11e18094d..90cf37e3f 100755
--- a/tools/kolla-images.py
+++ b/tools/kolla-images.py
@@ -221,7 +221,7 @@ def get_openstack_release() -> str:
 
 def validate(kolla_image_tags: KollaImageTags):
     """Validate the kolla_image_tags variable."""
-    tag_var_re = re.compile(r"^[a-z0-9_-]+$")
+    tag_var_re = re.compile(r"^[a-z0-9_]+$")
     openstack_release = get_openstack_release()
     tag_res = {
         base_distro: re.compile(f"^{openstack_release}-{base_distro}-[\d]{{8}}T[\d]{{6}}$")

From 210d2f5568214a9a931cf69157c431ffe2da1f39 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 23:07:44 +0000
Subject: [PATCH 22/25] Real fix for haproxy-ssh

---
 etc/kayobe/kolla-image-tags.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index 6411078b8..a52d7e270 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -12,7 +12,7 @@ kolla_image_tags:
   cloudkitty:
     rocky-9: 2023.1-rocky-9-20231115T110235
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231115T110235
-  haproxy-ssh:
+  haproxy_ssh:
     rocky-9: 2023.1-rocky-9-20240104T071640
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
   letsencrypt:

From 5c2aecb7c9e994d7621f48cb8aaa0feb9a1bd708 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 2 Feb 2024 23:10:44 +0000
Subject: [PATCH 23/25] Revert change to automation

---
 .automation | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.automation b/.automation
index 9ba253a9a..b00f285be 160000
--- a/.automation
+++ b/.automation
@@ -1 +1 @@
-Subproject commit 9ba253a9adba51011ce74aa87e708015dafc75fc
+Subproject commit b00f285be240e34c643c4bd93a877e56587f71fa

From 7b5f902f14a5fa39a89ab7677b8bef9af060b9fe Mon Sep 17 00:00:00 2001
From: Will Szumski <williamszumski@gmail.com>
Date: Mon, 5 Feb 2024 09:38:41 +0000
Subject: [PATCH 24/25] Update previous release

---
 .github/workflows/stackhpc-all-in-one.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 9b437dcfc..8ed5095bf 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -81,9 +81,9 @@ jobs:
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
       # NOTE(upgrade): Reference the PREVIOUS release here.
-      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-yoga
+      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-zed
       # NOTE(upgrade): Reference the PREVIOUS release branch here.
-      PREVIOUS_BRANCH: stackhpc/yoga
+      PREVIOUS_BRANCH: stackhpc/zed
     steps:
       # If testing upgrade, checkout previous release, otherwise checkout current branch
       - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config

From e9c68d40ae7c2209f22b7b4469131066cdd8b1bb Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Mon, 5 Feb 2024 17:04:12 +0000
Subject: [PATCH 25/25] Add letsencrypt and haproxy-ssh images

---
 etc/kayobe/kolla-image-tags.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index b777534dd..b9fa290c1 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -12,8 +12,10 @@ kolla_image_tags:
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231115T110235
   haproxy_ssh:
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
+    rocky-9: 2023.1-rocky-9-20240205T162323
   letsencrypt:
     ubuntu-jammy: 2023.1-ubuntu-jammy-20240104T071640
+    rocky-9: 2023.1-rocky-9-20240205T162323
   neutron:
     rocky-9: 2023.1-rocky-9-20240202T145927
     ubuntu-jammy: 2023.1-ubuntu-jammy-20231220T222020