From ce924149b618cb05fea0b06c97c23cda9f2a9a2c Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 8 Dec 2023 16:46:50 +0000
Subject: [PATCH 01/25] Post service deploy hook for OpenStack Capacity

---
 doc/source/configuration/monitoring.rst       | 51 +++++++------------
 .../ansible/deploy-os-capacity-exporter.yml   | 22 +++++++-
 .../templates/os_capacity-clouds.yml.j2       |  8 +--
 .../post.d/deploy-os-capacity-exporter.yml    |  1 +
 etc/kayobe/stackhpc-monitoring.yml            |  8 +--
 ...capacity-deploy-hook-b52e87c0819df6fd.yaml |  9 ++++
 6 files changed, 54 insertions(+), 45 deletions(-)
 create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml
 create mode 100644 releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml

diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst
index 819da9769..f23c7a915 100644
--- a/doc/source/configuration/monitoring.rst
+++ b/doc/source/configuration/monitoring.rst
@@ -141,36 +141,26 @@ OpenStack Capacity
 ==================
 
 OpenStack Capacity allows you to see how much space you have available
-in your cloud. StackHPC Kayobe Config includes a playbook for manual
-deployment, and it's necessary that some variables are set before
-running this playbook.
+in your cloud. StackHPC Kayobe Config will deploy OpenStack Capacity
+by default on a service deploy, this can be disabled by setting
+``stackhpc_enable_os_capacity`` to false.
 
-To successfully deploy OpenStack Capacity, you are required to specify
-the OpenStack application credentials in ``kayobe/secrets.yml`` as:
+OpenStack Capacity is deployed automatically using a service deploy hook
+with the generated kolla-ansible admin credentials, you can override these
+by setting the authentication url, username, password, project name and
+project domain name in ``stackhpc-monitoring.yml``:
 
 .. code-block:: yaml
 
-    secrets_os_capacity_credential_id: <some_credential_id>
-    secrets_os_capacity_credential_secret: <some_credential_secret>
+    stackhpc_os_capacity_auth_url: <keystone_auth_url>
+    stackhpc_os_capacity_username: <openstack_username>
+    stackhpc_os_capacity_password: <openstack_password_encrypted_with_vault>
+    stackhpc_os_capacity_project_name: <openstack_project_name>
+    stackhpc_os_capacity_domain_name: <openstack_project_domain_name>
+    stackhpc_os_capacity_openstack_region_name: <openstack_region_name>
 
-The Keystone authentication URL and OpenStack region can be changed
-from their defaults in ``stackhpc-monitoring.yml`` should you need to
-set a different OpenStack region for your cloud. The authentication
-URL is set to use ``kolla_internal_fqdn`` by default:
-
-.. code-block:: yaml
-
-    stackhpc_os_capacity_auth_url: <some_authentication_url>
-    stackhpc_os_capacity_openstack_region_name: <some_openstack_region>
-
-Additionally, you are required to enable a conditional flag to allow
-HAProxy and Prometheus configuration to be templated during deployment.
-
-.. code-block:: yaml
-
-    stackhpc_enable_os_capacity: true
-
-If you are deploying in a cloud with internal TLS, you may be required
+Additionally, you should ensure these credentials have the correct permissions
+for the exporter. If you are deploying in a cloud with internal TLS, you may be required
 to disable certificate verification for the OpenStack Capacity exporter
 if your certificate is not signed by a trusted CA.
 
@@ -178,21 +168,14 @@ if your certificate is not signed by a trusted CA.
 
     stackhpc_os_capacity_openstack_verify: false
 
-After defining your credentials, you may deploy OpenStack Capacity
-using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook
+If you've modified your credentials, you will need to re-deploy OpenStack Capacity
+using the ``deploy-os-capacity-exporter.yml`` Ansible playbook
 via Kayobe.
 
 .. code-block:: console
 
     kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml
 
-It is required that you re-configure the Prometheus, Grafana and HAProxy
-services following deployment, to do this run the following Kayobe command.
-
-.. code-block:: console
-
-    kayobe overcloud service reconfigure -kt grafana,prometheus,loadbalancer
-
 If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus
 alerts after deployment it's likely the os_exporter secrets have not been
 set correctly, double check you have entered the correct authentication
diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
index 8cff6a89e..978c13e62 100644
--- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
+++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
@@ -17,14 +17,33 @@
       ansible.builtin.file:
         path: /opt/kayobe/os-capacity/
         state: directory
+      when: stackhpc_enable_os_capacity
+
+    - name: Read admin-openrc credential file
+      ansible.builtin.command:
+        cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh"
+      delegate_to: localhost
+      register: credential
+      when: stackhpc_enable_os_capacity
+
+    - name: Set facts for admin credentials
+      ansible.builtin.set_fact:
+        stackhpc_os_capacity_auth_url: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_project_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_domain_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_openstack_region_name: "{{ credential.stdout_lines | select('match', '.*OS_REGION_NAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_username: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\",'') }}"
+        stackhpc_os_capacity_password: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\",'') }}"
+      when: stackhpc_enable_os_capacity
 
     - name: Template clouds.yml
       ansible.builtin.template:
         src: templates/os_capacity-clouds.yml.j2
         dest: /opt/kayobe/os-capacity/clouds.yaml
+      when: stackhpc_enable_os_capacity
 
     - name: Ensure os_capacity container is running
-      docker_container:
+      community.docker.docker_container:
         name: os_capacity
         image: ghcr.io/stackhpc/os-capacity:master
         env:
@@ -37,3 +56,4 @@
         network_mode: host
         restart_policy: unless-stopped
       become: true
+      when: stackhpc_enable_os_capacity
diff --git a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2
index a821d6dcb..ef3c8d7a5 100644
--- a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2
+++ b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2
@@ -2,12 +2,14 @@ clouds:
   openstack:
     auth:
       auth_url: "{{ stackhpc_os_capacity_auth_url }}"
-      application_credential_id: "{{ secrets_os_capacity_credential_id }}"
-      application_credential_secret: "{{ secrets_os_capacity_credential_secret }}"
+      project_name: "{{ stackhpc_os_capacity_project_name }}"
+      domain_name: "{{ stackhpc_os_capacity_domain_name }}"
+      username: "{{ stackhpc_os_capacity_username }}"
+      password: "{{ stackhpc_os_capacity_password }}"
     region_name: "{{ stackhpc_os_capacity_openstack_region_name }}"
     interface: "internal"
     identity_api_version: 3
-    auth_type: "v3applicationcredential"
+    auth_type: "password"
 {% if not stackhpc_os_capacity_openstack_verify | bool %}
     verify: False
 {% endif %}
diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml
new file mode 120000
index 000000000..0cc70aace
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml
@@ -0,0 +1 @@
+../../../ansible/deploy-os-capacity-exporter.yml
\ No newline at end of file
diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml
index 13bf6ba0f..f08e552c3 100644
--- a/etc/kayobe/stackhpc-monitoring.yml
+++ b/etc/kayobe/stackhpc-monitoring.yml
@@ -14,13 +14,7 @@ alertmanager_low_memory_threshold_gib: 5
 # Whether the OpenStack Capacity exporter is enabled.
 # Enabling this flag will result in HAProxy configuration and Prometheus scrape
 # targets being templated during deployment.
-stackhpc_enable_os_capacity: false
-
-# Keystone authentication URL for OpenStack Capacity
-stackhpc_os_capacity_auth_url: "http{% if kolla_enable_tls_internal | bool %}s{% endif %}://{{ kolla_internal_fqdn }}:5000"
-
-# OpenStack region for OpenStack Capacity
-stackhpc_os_capacity_openstack_region_name: "RegionOne"
+stackhpc_enable_os_capacity: true
 
 # Whether TLS certificate verification is enabled for the OpenStack Capacity
 # exporter during Keystone authentication.
diff --git a/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml
new file mode 100644
index 000000000..547939199
--- /dev/null
+++ b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Automatic deployment for OpenStack Capacity via a Kayobe service
+    deploy hook using kolla admin credentials.
+upgrade:
+  - |
+    OpenStack Capacity no longer uses application credentials. Please
+    delete any previously generated application credentials.
\ No newline at end of file

From 2ec32ceaf155d1876ed15e7849386d44c373683b Mon Sep 17 00:00:00 2001
From: scrungus <tyler@stackhpc.com>
Date: Fri, 8 Mar 2024 15:39:16 +0000
Subject: [PATCH 02/25] bump magnum-capi-helm version

---
 etc/kayobe/kolla.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 8844a3cbd..ebf8b0929 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -415,7 +415,7 @@ kolla_build_blocks:
   magnum_base_footer: |
     RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | head -n -1 | bash
     {% raw %}
-    {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.10.0'] %}
+    {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.11.0'] %}
     RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }}
     {% endraw %}
 # Dict mapping image customization variable names to their values.

From 8d7077f286dfe37b540e1f9cd454677f44b30ac5 Mon Sep 17 00:00:00 2001
From: scrungus <tyler@stackhpc.com>
Date: Fri, 8 Mar 2024 15:43:41 +0000
Subject: [PATCH 03/25] reno

---
 .../notes/bump-magnum-capi-helm-6723d89456e6a590.yaml         | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml

diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml
new file mode 100644
index 000000000..864edf44b
--- /dev/null
+++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml
@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    Updates Magnum CAPI Helm driver version to v0.11.0

From e054b4de9465beb0817f19febcb0fd325537bc91 Mon Sep 17 00:00:00 2001
From: scrungus <tyler@stackhpc.com>
Date: Wed, 13 Mar 2024 17:17:49 +0000
Subject: [PATCH 04/25] bump tag

---
 etc/kayobe/kolla/globals.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 419b1ba72..03e17ae72 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -20,9 +20,9 @@ kayobe_image_tags:
     rocky: yoga-20231218T141822
     ubuntu: yoga-20231107T165648
   magnum:
-    centos: yoga-20240229T120519
-    rocky: yoga-20240229T120519
-    ubuntu: yoga-20240229T120519
+    centos: yoga-20240308T154440
+    rocky: yoga-20240308T154440
+    ubuntu: yoga-20240308T154440
   neutron:
     centos: yoga-20231114T125927
     rocky: yoga-20240105T120257

From f1564f4a7cf324d370ab6d19a6beb8ec71bf2a5c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 15 Mar 2024 11:35:08 +0000
Subject: [PATCH 05/25] hotfix: Fix setting containers_list and running without
 a command

---
 etc/kayobe/ansible/hotfix-containers.yml    | 2 +-
 etc/kayobe/ansible/run-container-hotfix.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml
index b6a811801..23c28a6b9 100644
--- a/etc/kayobe/ansible/hotfix-containers.yml
+++ b/etc/kayobe/ansible/hotfix-containers.yml
@@ -30,7 +30,7 @@
 
     - name: Set fact for containers list
       set_fact:
-        containers_list: host_containers.stdout
+        containers_list: "{{ host_containers.stdout }}"
 
     - name: Fail if no containers match given regex
       vars:
diff --git a/etc/kayobe/ansible/run-container-hotfix.yml b/etc/kayobe/ansible/run-container-hotfix.yml
index 582ade5da..de652e451 100644
--- a/etc/kayobe/ansible/run-container-hotfix.yml
+++ b/etc/kayobe/ansible/run-container-hotfix.yml
@@ -20,3 +20,4 @@
 
 - name: Run container_hotfix_command
   command: "{{ kolla_container_engine | default('docker')}} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}"
+  when: container_hotfix_command

From 37b387aa3888661b725009e6021676030dbb5bd4 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 15 Mar 2024 11:40:24 +0000
Subject: [PATCH 06/25] hotfix: Fix failure message

---
 etc/kayobe/ansible/hotfix-containers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml
index 23c28a6b9..677105f3e 100644
--- a/etc/kayobe/ansible/hotfix-containers.yml
+++ b/etc/kayobe/ansible/hotfix-containers.yml
@@ -36,7 +36,7 @@
       vars:
         hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}"
       fail:
-        msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers | split('\n') }}"
+        msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers.stdout_lines }}"
       when: hotfix_containers == ""
 
     - name: Ensure hotfix-files directory exists on the remote host

From 7067c92e460787a013b1a66ea09a4d73fa4a959a Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 18 Mar 2024 13:42:06 +0000
Subject: [PATCH 07/25] Merge pull request #981 from
 stackhpc/use-fork-requirements

Use StackHPC downstream requirements fork
---
 etc/kayobe/kolla/kolla-build.conf | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf
index 2cd9c7a25..d78d0ebe2 100644
--- a/etc/kayobe/kolla/kolla-build.conf
+++ b/etc/kayobe/kolla/kolla-build.conf
@@ -9,3 +9,8 @@ base_tag = focal-20231003
 base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
 {% endif %}
 build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }}
+
+[openstack-base]
+type = git
+location = https://github.com/stackhpc/requirements
+reference = stackhpc/{{ openstack_release }}

From 4ee0f70e8de5d9036b1e0c5db8dfe06bd708ca81 Mon Sep 17 00:00:00 2001
From: scrungus <tyler@stackhpc.com>
Date: Wed, 20 Mar 2024 12:54:01 +0000
Subject: [PATCH 08/25] feature reno

---
 releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml
index 864edf44b..7fc3cca1a 100644
--- a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml
+++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml
@@ -1,4 +1,4 @@
 ---
-upgrade:
+features:
   - |
     Updates Magnum CAPI Helm driver version to v0.11.0

From d68a23fe0c7a75c7ac927ac8c6ac1b4c89b6a262 Mon Sep 17 00:00:00 2001
From: Bartosz Bezak <bartosz@stackhpc.com>
Date: Wed, 20 Mar 2024 11:36:13 +0100
Subject: [PATCH 09/25] Update cephadm collection version

---
 etc/kayobe/ansible/requirements.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index aa28f43f6..92c3faecd 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -1,7 +1,7 @@
 ---
 collections:
   - name: stackhpc.cephadm
-    version: 1.14.0
+    version: 1.15.1
   # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the
   # pulp_glue Python library being installed.
   - name: pulp.squeezer

From 9f6a0173b30804b71de4e0fac566cf106b49b89b Mon Sep 17 00:00:00 2001
From: Seunghun Lee <seunghun@stackhpc.com>
Date: Wed, 20 Mar 2024 13:46:29 +0000
Subject: [PATCH 10/25] Rebuild heat images with yaql 3.0.0 for zed

---
 etc/kayobe/kolla-image-tags.yml                            | 3 +++
 .../rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml     | 7 +++++++
 2 files changed, 10 insertions(+)
 create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index a264fdbf1..c26b70d7d 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -6,6 +6,9 @@ kolla_image_tags:
   openstack:
     rocky-9: zed-rocky-9-20240202T105829
     ubuntu-jammy: zed-ubuntu-jammy-20240129T151534
+  heat:
+    rocky-9: zed-rocky-9-20240320T113114
+    ubuntu-jammy: zed-ubuntu-jammy-20240320T113114
   magnum:
     rocky-9: zed-rocky-9-20240301T100039
     ubuntu-jammy: zed-ubuntu-jammy-20240301T100039
diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml
new file mode 100644
index 000000000..da3cb5cbb
--- /dev/null
+++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml
@@ -0,0 +1,7 @@
+---
+security:
+  - |
+    The Heat container images are rebuilt with yaql 3.0.0 to include patch for
+    vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat
+    services in your system with the current version of Heat images from
+    StackHPC Release Train.

From 91cc9ec5883f791067e2538e23581bab76eda8ad Mon Sep 17 00:00:00 2001
From: Seunghun Lee <seunghun@stackhpc.com>
Date: Wed, 20 Mar 2024 13:51:39 +0000
Subject: [PATCH 11/25] Rebuild heat images with yaql 3.0.0 for yoga

---
 etc/kayobe/kolla/globals.yml                               | 5 +++++
 .../rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml     | 7 +++++++
 2 files changed, 12 insertions(+)
 create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 03e17ae72..e860121d8 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -19,6 +19,10 @@ kayobe_image_tags:
     centos: yoga-20231107T165648
     rocky: yoga-20231218T141822
     ubuntu: yoga-20231107T165648
+  heat:
+    centos: yoga-20240320T082414
+    rocky: yoga-20240320T082414
+    ubuntu: yoga-20240320T082414
   magnum:
     centos: yoga-20240308T154440
     rocky: yoga-20240308T154440
@@ -33,6 +37,7 @@ kayobe_image_tags:
     ubuntu: yoga-20231103T161400
 
 cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
+heat_tag: "{% raw %}{{ kayobe_image_tags['heat'][kolla_base_distro] }}{% endraw %}"
 magnum_tag: "{% raw %}{{ kayobe_image_tags['magnum'][kolla_base_distro] }}{% endraw %}"
 neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}"
 nova_tag: "{% raw %}{{ kayobe_image_tags['nova'][kolla_base_distro] }}{% endraw %}"
diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml
new file mode 100644
index 000000000..da3cb5cbb
--- /dev/null
+++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml
@@ -0,0 +1,7 @@
+---
+security:
+  - |
+    The Heat container images are rebuilt with yaql 3.0.0 to include patch for
+    vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat
+    services in your system with the current version of Heat images from
+    StackHPC Release Train.

From 9beb7fb09bb5c51a9261e7bed73564378f5e854e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 21 Mar 2024 13:18:05 +0000
Subject: [PATCH 12/25] Fail on any unparsed Ansible inventory

If Ansible is unable to parse an inventory source, by default it will
print a warning and continue execution. Typically this highlights an
important error that should be addressed.

This change modifies the Ansible configuration to error out in this
case.
---
 etc/kayobe/ansible.cfg                                     | 4 ++++
 .../notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml    | 7 +++++++
 2 files changed, 11 insertions(+)
 create mode 100644 releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml

diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg
index 901001ad8..b38cb8239 100644
--- a/etc/kayobe/ansible.cfg
+++ b/etc/kayobe/ansible.cfg
@@ -11,5 +11,9 @@ callbacks_enabled = ansible.posix.profile_tasks
 # Silence warning about invalid characters found in group names
 force_valid_group_names = ignore
 
+[inventory]
+# Fail when any inventory source cannot be parsed.
+any_unparsed_is_failed = True
+
 [ssh_connection]
 pipelining = True
diff --git a/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml
new file mode 100644
index 000000000..335691c30
--- /dev/null
+++ b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml
@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    Updates the Ansible configuration to `fail on any unparsed inventory source
+    <https://docs.ansible.com/ansible/latest/reference_appendices/config.html#inventory-any-unparsed-is-failed>`__.
+    If you are using a separate Ansible configuration for Kolla Ansible, you
+    may wish to add this setting in ``etc/kayobe/kolla/ansible.cfg``.

From 99838a89987a6c4250f4eb0a55ed7d199853b5fe Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 26 Mar 2024 12:42:46 +0000
Subject: [PATCH 13/25] docs: Add an upgrade doc note about Glance
 show_multiple_locations

Operators may decide to replace this option.
---
 doc/source/operations/upgrading.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst
index 53df5aef2..d44f1e917 100644
--- a/doc/source/operations/upgrading.rst
+++ b/doc/source/operations/upgrading.rst
@@ -94,6 +94,20 @@ custom configuration that uses the Keystone admin port. One such example is the
 config for Ceph RGW in ``etc/kayobe/cephadm.yml``. Be sure to update any manual
 references to the old port.
 
+Glance show_multiple_locations disabled
+---------------------------------------
+
+Kolla Ansible no longer sets ``show_multiple_locations = True`` in Glance by
+default when Glance's Ceph RBD backend is enabled. This was applied as a fix
+but operators must note that this, in turn, disables Cinder and Nova's
+optimisations. In particular, this can increase instance creation times due to
+a lack of copy-on-write.
+
+On the other hand, these optimisations might have been causing other trouble
+for operators. Please see `LP#1992153
+<https://bugs.launchpad.net/kolla-ansible/+bug/1992153>`__. Operators relying
+on this feature can set the flag themselves using service config overrides.
+
 Known issues
 ============
 

From 8f6fcb81af1f53355056ed511d2b18860a2d2125 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 28 Mar 2024 14:15:23 +0000
Subject: [PATCH 14/25] Fix host image builds on Arc runners

Arc runners are kubernetes-orchestrated github runners. Host image
builds do not work on these runners, so this commit adapts the host
image build workflow to spin up a worker VM which executes the build.
---
 .../workflows/overcloud-host-image-build.yml  | 400 +++++++++++-------
 .github/workflows/stackhpc-ci-cleanup.yml     |  20 +
 .../ansible/openstack-host-image-upload.yml   |  54 +++
 etc/kayobe/ansible/pulp-host-image-upload.yml |  16 +-
 .../environments/ci-builder/inventory/hosts   |   2 +-
 etc/kayobe/overcloud-dib.yml                  |   2 +-
 6 files changed, 323 insertions(+), 171 deletions(-)
 create mode 100644 etc/kayobe/ansible/openstack-host-image-upload.yml

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 4c338cda3..1b6c2e5f9 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -35,21 +35,38 @@ on:
 
 env:
   ANSIBLE_FORCE_COLOR: True
+  KAYOBE_ENVIRONMENT: ci-builder
+  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-host-image-build:
     name: Build overcloud host images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: arc-skc-host-image-builder-runner
     permissions: {}
     steps:
-      - uses: actions/checkout@v4
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
+
+      - name: Start the SSH service
+        run: |
+          sudo /etc/init.d/ssh start
+
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           path: src/kayobe-config
 
+      - name: Output image tag of the builder
+        id: builder_image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+
       - name: Determine OpenStack release
         id: openstack_release
         run: |
-          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
       # Generate a tag to apply to all built overcloud host images.
@@ -62,10 +79,6 @@ jobs:
         run: |
           echo "${{ steps.host_image_tag.outputs.host_image_tag }}"
 
-      - name: Clean any previous build artifact
-        run: |
-          rm -f /tmp/updated_images.txt
-
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -73,34 +86,6 @@ jobs:
           ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
           path: src/kayobe
 
-      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
-      - name: Install dependencies
-        run: |
-          sudo dnf -y install python3-virtualenv zstd
-
-      - name: Setup networking
-        run: |
-          if ! ip l show breth1 >/dev/null 2>&1; then
-              sudo ip l add breth1 type bridge
-          fi
-          sudo ip l set breth1 up
-          if ! ip a show breth1 | grep 192.168.33.3/24; then
-              sudo ip a add 192.168.33.3/24 dev breth1
-          fi
-          if ! ip l show dummy1 >/dev/null 2>&1; then
-              sudo ip l add dummy1 type dummy
-          fi
-          sudo ip l set dummy1 up
-          sudo ip l set dummy1 master breth1
-
-      # FIXME: Without this workaround we see the following issue after the runner is power cycled:
-      # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] ***
-      #   Unable to start service network: Job for network.service failed because the control process exited with error code.
-      #   See \"systemctl status network.service\" and \"journalctl -xe\" for details.
-      - name: Kill dhclient (workaround)
-        run: |
-          (sudo killall dhclient || true) && sudo systemctl restart network
-
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -110,36 +95,132 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Initialise terraform
+        run: terraform init
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Generate SSH keypair
+        run: ssh-keygen -f id_rsa -N ''
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Generate clouds.yaml
+        run: |
+          cat << EOF > clouds.yaml
+          ${{ secrets.CLOUDS_YAML }}
+          EOF
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Generate terraform.tfvars
+        run: |
+          cat << EOF > terraform.tfvars
+          ssh_public_key = "id_rsa.pub"
+          ssh_username = "rocky"
+          aio_vm_name = "skc-host-image-builder"
+          # Must be a Rocky Linux 9 host to successfully build all images
+          # This MUST NOT be an LVM image. It can cause confusing conficts with the built image.
+          aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2"
+          aio_vm_flavor = "en1.medium"
+          aio_vm_network = "stackhpc-ci"
+          aio_vm_subnet = "stackhpc-ci"
+          aio_vm_interface = "eth0"
+          EOF
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Terraform Plan
+        run: terraform plan
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Terraform Apply
+        run: |
+          for attempt in $(seq 5); do
+              if terraform apply -auto-approve; then
+                  echo "Created infrastructure on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to create infrastructure on attempt $attempt"
+              sleep 10
+              terraform destroy -auto-approve
+              sleep 60
+          done
+          echo "Failed to create infrastructure after $attempt attempts"
+          exit 1
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Get Terraform outputs
+        id: tf_outputs
+        run: |
+          terraform output -json
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+
+      - name: Write Terraform outputs
+        run: |
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
+          ${{ steps.tf_outputs.outputs.stdout }}
+          EOF
+
+      - name: Write Terraform network config
+        run: |
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
+          ---
+          aio_ips:
+            builder: "{{ access_ip_v4.value }}"
+          EOF
+
+      - name: Write Terraform network interface config
+        run: |
+          mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
+          rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
+          cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
+          admin_interface: "{{ access_interface.value }}"
+          aio_interface: "{{ access_interface.value }}"
+          EOF
+
+      - name: Manage SSH keys
+        run: |
+          mkdir -p ~/.ssh
+          touch ~/.ssh/authorized_keys
+          cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
+          cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
+
       - name: Bootstrap the control host
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe control host bootstrap
 
-      - name: Configure the seed host
+      - name: Configure the seed host (Builder VM)
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure
+          kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network
+
+      - name: Install dependencies
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run \
+          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Create bifrost_httpboot Docker volume
-        run: |
-          if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then
-            sudo docker volume create bifrost_httpboot
-          fi
-
-      - name: Generate clouds.yaml
-        run: |
-          cat << EOF > clouds.yaml
-          ${{ secrets.CLOUDS_YAML }}
-          EOF
-
-      - name: Install OpenStack client
         run: |
           source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Build a CentOS Stream 8 overcloud host image
         id: build_centos_stream_8
@@ -155,6 +236,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.centos
 
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: steps.build_centos_stream_8.outcome == 'failure'
+
       - name: Upload CentOS Stream 8 overcloud host image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
@@ -169,18 +260,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'
 
-      - name: Upload CentOS Stream 8 overcloud host image to SMS
+      - name: Upload CentOS Stream 8 overcloud host image to Dev Cloud
         run: |
           source venvs/kayobe/bin/activate &&
-          openstack image create \
-          overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} \
-          --container-format bare \
-          --disk-format qcow2 \
-          --file /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2 \
-          --private \
-          --os-cloud sms-lab-release \
-          --progress
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
+          -e local_image_path="/opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2" \
+          -e image_name=overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }}
         env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'
@@ -199,6 +288,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky8
 
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: steps.build_rocky_8.outcome == 'failure'
+
       - name: Upload Rocky Linux 8 overcloud host image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
@@ -212,19 +311,17 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'
-        
-      - name: Upload Rocky Linux 8 overcloud host image to SMS
+
+      - name: Upload Rocky Linux 8 overcloud host image to Dev Cloud
         run: |
           source venvs/kayobe/bin/activate &&
-          openstack image create \
-          overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \
-          --container-format bare \
-          --disk-format qcow2 \
-          --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \
-          --private \
-          --os-cloud sms-lab-release \
-          --progress
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
+          -e local_image_path="/opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2" \
+          -e image_name=overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }}
         env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'
@@ -243,6 +340,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9
 
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: steps.build_rocky_9.outcome == 'failure'
+
       - name: Upload Rocky Linux 9 overcloud host image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
@@ -256,19 +363,17 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
-        
-      - name: Upload Rocky Linux 9 overcloud host image to SMS
+
+      - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud
         run: |
           source venvs/kayobe/bin/activate &&
-          openstack image create \
-          overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \
-          --container-format bare \
-          --disk-format qcow2 \
-          --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \
-          --private \
-          --os-cloud sms-lab-release \
-          --progress
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
+          -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \
+          -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }}
         env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
@@ -287,6 +392,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu-focal
 
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: steps.build_ubuntu_focal.outcome == 'failure'
+
       - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
@@ -301,18 +416,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'
 
-      - name: Upload Ubuntu Focal 20.04 overcloud host image to SMS
+      - name: Upload Ubuntu Focal overcloud host image to Dev Cloud
         run: |
           source venvs/kayobe/bin/activate &&
-          openstack image create \
-          overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \
-          --container-format bare \
-          --disk-format qcow2 \
-          --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \
-          --private \
-          --os-cloud sms-lab-release \
-          --progress
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
+          -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2" \
+          -e image_name=overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }}
         env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'
@@ -331,6 +444,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu-jammy
 
+      - name: Show last error logs
+        continue-on-error: true
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: steps.build_ubuntu_jammy.outcome == 'failure'
+
       - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark
         run: |
           source venvs/kayobe/bin/activate &&
@@ -345,83 +468,27 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
-      - name: Upload Ubuntu Jammy 22.04 overcloud host image to SMS
+      - name: Upload Ubuntu Jammy overcloud host image to Dev Cloud
         run: |
           source venvs/kayobe/bin/activate &&
-          openstack image create \
-          overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \
-          --container-format bare \
-          --disk-format qcow2 \
-          --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \
-          --private \
-          --os-cloud sms-lab-release \
-          --progress
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run \
+          src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
+          -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \
+          -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }}
         env:
+          CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
-      - name: Upload updated images artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: Updated images list
-          path: /tmp/updated_images.txt
-          retention-days: 7
-        if: steps.build_centos_stream_8.outcome == 'success' ||
-            steps.build_rocky_8.outcome == 'success' ||
-            steps.build_rocky_9.outcome == 'success' ||
-            steps.build_ubuntu_focal.outcome == 'success' ||
-            steps.build_ubuntu_jammy.outcome == 'success'
-
-      - name: Upload CentOS build logs if build failed
-        uses: actions/upload-artifact@v4
-        with:
-          name: CentOS build logs
-          path: |
-            /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout
-            /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stderr
-          retention-days: 7
-        if: steps.build_centos_stream_8.outcome == 'failure'
-
-      - name: Upload Rocky 8 build logs if build failed
-        uses: actions/upload-artifact@v4
-        with:
-          name: Rocky 8 build logs
-          path: |
-            /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout
-            /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stderr
-          retention-days: 7
-        if: steps.build_rocky_8.outcome == 'failure'
-
-      - name: Upload Rocky 9 build logs if build failed
-        uses: actions/upload-artifact@v4
-        with:
-          name: Rocky 9 build logs
-          path: |
-            /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout
-            /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr
-          retention-days: 7
-        if: steps.build_rocky_9.outcome == 'failure'
-
-      - name: Upload Ubuntu Focal 20.04 build logs if build failed
-        uses: actions/upload-artifact@v4
-        with:
-          name: Ubuntu Focal 20.04 build logs
-          path: |
-            /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout
-            /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stderr
-          retention-days: 7
-        if: steps.build_ubuntu_focal.outcome == 'failure'
-
-      - name: Upload Ubuntu Jammy 22.04 build logs if build failed
-        uses: actions/upload-artifact@v4
-        with:
-          name: Ubuntu Jammy 22.04 build logs
-          path: |
-            /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout
-            /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr
-          retention-days: 7
-        if: steps.build_ubuntu_jammy.outcome == 'failure'
+      - name: Copy logs back
+        continue-on-error: true
+        run: |
+          mkdir logs
+          scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/
+          scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true
+        if: always()
 
       - name: Fail if any overcloud host image builds failed
         run: |
@@ -433,7 +500,18 @@ jobs:
             steps.build_ubuntu_focal.outcome == 'failure' ||
             steps.build_ubuntu_jammy.outcome == 'failure'
 
-      - name: Clean up build artifacts
-        run: |
-          sudo rm -rf /opt/kayobe/images/
+      - name: Upload logs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: Build logs
+          path: ./logs
         if: always()
+
+      - name: Destroy
+        run: terraform destroy -auto-approve
+        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+        if: always()
\ No newline at end of file
diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml
index a769aa718..ed9ec327c 100644
--- a/.github/workflows/stackhpc-ci-cleanup.yml
+++ b/.github/workflows/stackhpc-ci-cleanup.yml
@@ -55,3 +55,23 @@ jobs:
           OS_CLOUD: openstack
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: Clean up host image builder instances over 5 hours old
+        run: |
+          result=0
+          changes_before=$(date -Imin -d -5hours)
+          for status in ACTIVE BUILD ERROR SHUTOFF; do
+              for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
+                  echo "Cleaning up $status instance $instance"
+                  openstack server show $instance
+                  if ! openstack server delete $instance; then
+                      echo "Failed to delete $status instance $instance"
+                      result=1
+                  fi
+              done
+          done
+          exit $result
+        env:
+          OS_CLOUD: openstack
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml
new file mode 100644
index 000000000..2c92d2446
--- /dev/null
+++ b/etc/kayobe/ansible/openstack-host-image-upload.yml
@@ -0,0 +1,54 @@
+---
+# This playbook is designed to be used by the overcloud-host-image-build.yml
+# GitHub workflow to upload newly-built images to a development cloud for
+# testing and use in CI.
+- name: Upload an OS image to Glance
+  hosts: seed
+  vars:
+    local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2"
+    image_name: "overcloud-{{ os_distribution }}-{{ os_release }}"
+  tasks:
+    - block:
+        - name: Write out clouds.yaml
+          copy:
+            content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}"
+            dest: clouds.yaml
+            mode: 0600
+
+        - name: Write out secure.yaml
+          no_log: true
+          vars:
+            os_secrets:
+              clouds:
+                openstack:
+                  auth:
+                    application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}"
+                    application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}"
+          copy:
+            content: "{{ os_secrets | to_nice_yaml }}"
+            dest: secure.yaml
+            mode: 0600
+
+        - name: Ensure dependencies are installed
+          pip:
+            name: openstacksdk
+
+        - name: Upload an image to Glance
+          openstack.cloud.image:
+            cloud: openstack
+            name: "{{ image_name }}"
+            container_format: bare
+            disk_format: qcow2
+            state: present
+            filename: "{{ local_image_path }}"
+
+      always:
+        - name: Remove clouds.yaml
+          file:
+            path: clouds.yaml
+            state: absent
+
+        - name: Remove secure.yaml
+          file:
+            path: secure.yaml
+            state: absent
diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml
index a06897d90..d3a44f133 100644
--- a/etc/kayobe/ansible/pulp-host-image-upload.yml
+++ b/etc/kayobe/ansible/pulp-host-image-upload.yml
@@ -1,12 +1,12 @@
 ---
 - name: Upload and create a distribution for an image
-  hosts: localhost
+  hosts: seed
   vars:
     remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
     remote_pulp_username: "{{ stackhpc_image_repository_username }}"
     remote_pulp_password: "{{ stackhpc_image_repository_password }}"
     repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
-    base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
+    pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
   tasks:
     - name: Print image tag
       debug:
@@ -74,7 +74,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_latest"
-        base_path: "{{ base_path }}/latest"
+        base_path: "{{ pulp_base_path }}/latest"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -86,7 +86,7 @@
         username: "{{ remote_pulp_username }}"
         password: "{{ remote_pulp_password }}"
         name: "{{ repository_name }}_{{ host_image_tag }}"
-        base_path: "{{ base_path }}/{{ host_image_tag }}"
+        base_path: "{{ pulp_base_path }}/{{ host_image_tag }}"
         publication: "{{ publication_details.publication.pulp_href }}"
         content_guard: development
         state: present
@@ -95,26 +95,26 @@
     - name: Update new images file with versioned path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
         create: true
 
     - name: Update new images file with latest path
       lineinfile:
         path: /tmp/updated_images.txt
-        line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print versioned path
       debug:
-        msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         {{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
     - name: Print latest path
       debug:
-        msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
+        msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
         latest/{{ found_files.files[0].path | basename }}"
       when: latest_distribution_details.changed
 
diff --git a/etc/kayobe/environments/ci-builder/inventory/hosts b/etc/kayobe/environments/ci-builder/inventory/hosts
index 49b7be166..33fda8b73 100644
--- a/etc/kayobe/environments/ci-builder/inventory/hosts
+++ b/etc/kayobe/environments/ci-builder/inventory/hosts
@@ -1,3 +1,3 @@
 # A 'seed' host used for building images.
 [seed]
-localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
+builder
diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml
index 8f59d58ef..d7f6dbd69 100644
--- a/etc/kayobe/overcloud-dib.yml
+++ b/etc/kayobe/overcloud-dib.yml
@@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra:
 overcloud_dib_git_elements_extra:
   - repo: "https://github.com/stackhpc/stackhpc-image-elements"
     local: "{{ source_checkout_path }}/stackhpc-image-elements"
-    version: "v1.6.0"
+    version: "v1.6.1"
     elements_path: "elements"
 
 # List of git repositories containing Diskimage Builder (DIB) elements. See

From c1a31acc9d8e85330b664f220051a39e1c2cb9ed Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 2 Apr 2024 14:56:45 +0100
Subject: [PATCH 15/25] Fix AIO connectivity loss in automated script

---
 etc/kayobe/environments/ci-aio/automated-setup.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh
index 5129db015..f5468a09a 100644
--- a/etc/kayobe/environments/ci-aio/automated-setup.sh
+++ b/etc/kayobe/environments/ci-aio/automated-setup.sh
@@ -84,6 +84,10 @@ kayobe overcloud host configure
 
 kayobe overcloud service deploy
 
+if type apt; then
+    sudo cp /run/systemd/network/* /etc/systemd/network
+fi
+
 export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config
 export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe
 pushd $BASE_PATH/src/kayobe

From faaabbb5aa92f547f48f513233fdf24785c9e7b7 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 4 Apr 2024 11:43:40 +0100
Subject: [PATCH 16/25] Fix AIO deploy script

---
 etc/kayobe/environments/ci-aio/automated-setup.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh
index f5468a09a..84b9b5f09 100644
--- a/etc/kayobe/environments/ci-aio/automated-setup.sh
+++ b/etc/kayobe/environments/ci-aio/automated-setup.sh
@@ -72,6 +72,10 @@ fi
 sudo ip l set dummy1 up
 sudo ip l set dummy1 master breth1
 
+if type apt; then
+    sudo cp /run/systemd/network/* /etc/systemd/network
+fi
+
 export KAYOBE_VAULT_PASSWORD=$(cat $BASE_PATH/vault-pw)
 pushd $BASE_PATH/src/kayobe-config
 source kayobe-env --environment ci-aio
@@ -84,10 +88,6 @@ kayobe overcloud host configure
 
 kayobe overcloud service deploy
 
-if type apt; then
-    sudo cp /run/systemd/network/* /etc/systemd/network
-fi
-
 export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config
 export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe
 pushd $BASE_PATH/src/kayobe

From 2d8d500925fd90b7cc83b4fd8af3f53329a2696e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 8 Apr 2024 11:24:55 +0100
Subject: [PATCH 17/25] ci-multinode: Use skc-ci-aio user for ci-multinode env

Similar to c338dd9b7cad77c14eb15eb0193d02b0c9ff78b4, but applied to
ci-multinode instead of ci-aio.

This user only has read-only access to the package and container
repositories, so is safer than using the release-train-ci user which has
read/write permissions.
---
 .../environments/ci-multinode/stackhpc-ci.yml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index cdb6eb810..ae5768bac 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -11,6 +11,14 @@ kolla_docker_namespace: stackhpc-dev
 # Host and port of a package repository mirror.
 # Build and deploy the development Pulp service repositories.
 stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080"
+stackhpc_repo_mirror_username: "skc-ci-aio"
+stackhpc_repo_mirror_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  36373536303261313239613761653261663437356566343865383563346334396136653666383765
+  6634396534653865633936653038383132396532386665370a366562383166353966663838316266
+  65333133636330623936623438666632316238376264313234346333346461623765633163353635
+  6565326136313564320a303231383438333062643533333335663034613439393665656162626137
+  65356232656164663831316530333136336362393636656566353635306565626636
 
 # Build and deploy released Pulp repository versions.
 stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
@@ -66,12 +74,5 @@ stackhpc_include_os_minor_version_in_repo_url: true
 # Push built images to the development Pulp service registry.
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
 
-# Username and password of container registry.
-stackhpc_docker_registry_username: "stackhpc-kayobe-ci"
-stackhpc_docker_registry_password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          33356166343730633865363431306535613736663764373034396132356131343066636530393534
-          3262646436663034633131316438633230383330633533350a386365313239303464383636376338
-          61656662333939333063343131633963636431663136643137636664633233633133396339613861
-          3038613063626138610a333566393937643630366564653163613364323965396130613433316537
-          39653335393831633362343934363866346262613166393561666336623062393935
+stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}"
+stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}"

From d77fcb1f64abbe50171d493df05e76e7f03e2a2e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 8 Apr 2024 15:29:00 +0100
Subject: [PATCH 18/25] ci-multinode: Use Ark package repositories to install
 packages

Similar to e9130b9c51161fdadd676932eae5f2c13f5948a8 but applied to
ci-multinode rather than ci-aio.

Previously we were using Test Pulp on SMS lab, but this is out of
action. Switching to Ark allows CI jobs to run on Leafcloud (or anywhere
with Internet access).
---
 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index ae5768bac..32f8775e1 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -10,7 +10,8 @@ kolla_docker_namespace: stackhpc-dev
 
 # Host and port of a package repository mirror.
 # Build and deploy the development Pulp service repositories.
-stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080"
+# Use Ark's package repositories to install packages.
+stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "skc-ci-aio"
 stackhpc_repo_mirror_password: !vault |
   $ANSIBLE_VAULT;1.1;AES256

From c57f2c3a7b93d17ed1ccfd31c8c596dd6c2e3064 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 8 Apr 2024 15:22:07 +0100
Subject: [PATCH 19/25] ci-multinode: Allow rebooting for SELinux state

The Yoga overcloud host images currently have SELinux disabled, but the
default config enables SELinux in permissive mode on Rocky Linux 9. This
change allows the ci-multinode environment to run on these images.
---
 etc/kayobe/environments/ci-multinode/globals.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml
index daecef4f2..fe7285f4c 100644
--- a/etc/kayobe/environments/ci-multinode/globals.yml
+++ b/etc/kayobe/environments/ci-multinode/globals.yml
@@ -64,7 +64,7 @@ stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id"
 ###############################################################################
 
 # Avoid a reboot.
-disable_selinux_do_reboot: false
+disable_selinux_do_reboot: true
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

From e2b2f40cd138a212b2801ed410f791003a0a5fd5 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 8 Apr 2024 15:31:37 +0100
Subject: [PATCH 20/25] ci-multinode: Add API FQDNs to /etc/hosts in
 fix-networking.yml

This avoids using the add-fqdn.yml playbook in
terraform-kayobe-multinode, which requires the Terraform/Ansible client
to have access to all hosts.
---
 etc/kayobe/ansible/fix-networking.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml
index 0b14f9ddf..8105db8f4 100644
--- a/etc/kayobe/ansible/fix-networking.yml
+++ b/etc/kayobe/ansible/fix-networking.yml
@@ -10,11 +10,13 @@
     # Work around no known_hosts entry on first boot.
     ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
   tasks:
-    - name: Ensure `hosts` file contains pulp entries
+    - name: Ensure `hosts` file contains pulp and API entries
       blockinfile:
         path: /etc/hosts
-        marker: "# {mark} Kayobe Pulp entries"
+        marker: "# {mark} Kayobe entries"
         block: |
           10.0.0.34 pelican pelican.service.compute.sms-lab.cloud
           10.205.3.187 pulp-server pulp-server.internal.sms-cloud
+          192.168.37.2 internal.infra.mos.{{ root_domain }}
+          192.168.39.2 public.infra.mos.{{ root_domain }}
       become: true

From 2ca68f1713b5784fcfc577f37d3b6528f8017060 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 10 Apr 2024 09:13:17 +0100
Subject: [PATCH 21/25] ci-multinode: Wait for connection in fix-networking.yml

This allows us to drop the fix-homedir-ownership.yml playbook in
terraform-kayobe-multinode, which also performed the function of waiting
for hosts to become reachable.
---
 etc/kayobe/ansible/fix-networking.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml
index 8105db8f4..01a833264 100644
--- a/etc/kayobe/ansible/fix-networking.yml
+++ b/etc/kayobe/ansible/fix-networking.yml
@@ -10,6 +10,9 @@
     # Work around no known_hosts entry on first boot.
     ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
   tasks:
+    - name: Ensure hosts are reachable
+      ansible.builtin.wait_for_connection:
+
     - name: Ensure `hosts` file contains pulp and API entries
       blockinfile:
         path: /etc/hosts

From 33c0d38622136cde0fd46458f837ecd35b18ac40 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 10 Apr 2024 09:12:05 +0100
Subject: [PATCH 22/25] ci-multinode: Use qemu virtualisation

Most multinode environments will use nested virtualisation, and we
can't guarantee that nested KVM support is available. Use QEMU as a
lowest common denominator.

We might consider setting this dynamically based on the hypervisor in
future.
---
 etc/kayobe/environments/ci-multinode/kolla/globals.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
index eab31a1d8..4f9506be0 100644
--- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml
+++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
@@ -1,4 +1,9 @@
 ---
+# Most development environments will use nested virtualisation, and we can't
+# guarantee that nested KVM support is available. Use QEMU as a lowest common
+# denominator.
+nova_compute_virt_type: qemu
+
 # Reduce the control plane's memory footprint by limiting the number of worker
 # processes to two per-service when running in a VM.
 openstack_service_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 5] | min }}{% endraw %}"

From eb1f88ec51ee65a504e064b99e3aadc4ab65cbe0 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 8 Apr 2024 16:03:51 +0100
Subject: [PATCH 23/25] ci-multinode: Set default Ceph release to Quincy on
 Rocky Linux 9

Pacific is not supported on Rocky Linux 9, so it does not make sense as a default.
---
 etc/kayobe/environments/ci-multinode/cephadm.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml
index 7885a5735..4a9d3f448 100644
--- a/etc/kayobe/environments/ci-multinode/cephadm.yml
+++ b/etc/kayobe/environments/ci-multinode/cephadm.yml
@@ -2,6 +2,12 @@
 ###############################################################################
 # Cephadm deployment configuration.
 
+# Ceph release name.
+cephadm_ceph_release: "{{ 'quincy' if (ansible_facts['distribution_release'] == 'jammy' or ansible_facts.distribution_major_version == '9') else 'pacific' }}"
+
+# Ceph container image tag.
+cephadm_image_tag: "{{ 'v17.2.7' if cephadm_ceph_release == 'quincy' else 'v16.2.14' }}"
+
 # Ceph OSD specification.
 cephadm_osd_spec:
   service_type: osd

From 50378160654a5ed5d74cb17180cb3999401d09b5 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 10 Apr 2024 09:09:23 +0100
Subject: [PATCH 24/25] os_capacity: Add tags to playbook, update vault docs

Previously the first deployment of a system with a Vault CA for internal
TLS and os_capacity enabled would fail when deploying HAProxy.
os_capacity deployment requires admin-openrc.sh to exist, but because of
the use of -kt haproxy the post-deploy tasks that create it will be
skipped.

This change fixes the issue by adding an os_capacity tag to the relevant
plays, and updating the Vault docs to skip the new tag when deploying
HAProxy.
---
 doc/source/configuration/vault.rst                 | 2 +-
 etc/kayobe/ansible/deploy-os-capacity-exporter.yml | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst
index 4cb39b61b..21268f108 100644
--- a/doc/source/configuration/vault.rst
+++ b/doc/source/configuration/vault.rst
@@ -111,7 +111,7 @@ Setup HAProxy config for Vault
 
    .. code-block::
 
-      kayobe overcloud service deploy -kt haproxy
+      kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy
 
 Setup Vault HA on the overcloud hosts
 -------------------------------------
diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
index 978c13e62..cc3afa7b0 100644
--- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
+++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml
@@ -1,6 +1,7 @@
 ---
 - name: Remove legacy os_exporter.cfg file
   hosts: network
+  tags: os_capacity
   gather_facts: false
   tasks:
     - name: Ensure legacy os_exporter.cfg config file is deleted
@@ -11,6 +12,7 @@
 
 - name: Deploy os-capacity exporter
   hosts: monitoring
+  tags: os_capacity
   gather_facts: false
   tasks:
     - name: Create os-capacity directory

From 55b343b63dc8fb07d053ec4d242f468e9cf05142 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 11 Apr 2024 13:54:09 +0100
Subject: [PATCH 25/25] Revert "docs: Add an upgrade doc note about Glance
 show_multiple_locations"

This reverts commit 99838a89987a6c4250f4eb0a55ed7d199853b5fe.

It applies to the Zed upgrade, not Antelope.
---
 doc/source/operations/upgrading.rst | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst
index a23495bd6..89f8f6aa8 100644
--- a/doc/source/operations/upgrading.rst
+++ b/doc/source/operations/upgrading.rst
@@ -162,20 +162,6 @@ environment. This can result in significant changes to the Kolla config. Take
 extra care when creating the Antelope branch of the kayobe-config and always
 check the config diff.
 
-Glance show_multiple_locations disabled
----------------------------------------
-
-Kolla Ansible no longer sets ``show_multiple_locations = True`` in Glance by
-default when Glance's Ceph RBD backend is enabled. This was applied as a fix
-but operators must note that this, in turn, disables Cinder and Nova's
-optimisations. In particular, this can increase instance creation times due to
-a lack of copy-on-write.
-
-On the other hand, these optimisations might have been causing other trouble
-for operators. Please see `LP#1992153
-<https://bugs.launchpad.net/kolla-ansible/+bug/1992153>`__. Operators relying
-on this feature can set the flag themselves using service config overrides.
-
 Known issues
 ============