From 4172693351ffccad360255a5aa6518577d577737 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Wed, 8 Nov 2023 09:12:34 +0000
Subject: [PATCH 01/27] CloudKitty fix for ES->OS migration

---
 etc/kayobe/kolla/globals.yml                                 | 1 +
 .../cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml  | 5 +++++
 2 files changed, 6 insertions(+)
 create mode 100644 releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 19dfa9e3d..51d2eb7df 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -20,6 +20,7 @@ kayobe_image_tags:
     # with signature: `Missing section footer for 0000:00:01.3/piix4_pm`. Test carefully before bumping.
     centos: yoga-20230718T112646
 
+cloudkitty_tag: yoga-20231107T165648
 nova_tag: yoga-20231103T161400
 nova_libvirt_tag: "{% raw %}{{ kayobe_image_tags['nova_libvirt'][kolla_base_distro] | default(nova_tag) }}{% endraw %}"
 
diff --git a/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml
new file mode 100644
index 000000000..818f415a4
--- /dev/null
+++ b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes the bulk API of CloudKitty so that it now supports the migration
+    from Elasticsearch to OpenSearch.

From 117c5b786571778a04e0f15eb7ab32d2fe8b4015 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Wed, 8 Nov 2023 16:22:25 +0000
Subject: [PATCH 02/27] Document the manual process to delete a broken LB

---
 doc/source/operations/octavia.rst | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/doc/source/operations/octavia.rst b/doc/source/operations/octavia.rst
index 57f3f6233..80d7bdd25 100644
--- a/doc/source/operations/octavia.rst
+++ b/doc/source/operations/octavia.rst
@@ -38,3 +38,55 @@ when building new images.
 
 To rollback an image update, simply delete the old image. The next newest image with
 a tag matching ``amp_image_tag`` will be selected.
+
+Manually deleting broken load balancers
+=======================================
+
+Sometimes, a load balancer will get stuck in a broken state of ``PENDING_CREATE`` or ``PENDING_UPDATE``.
+When in this state, the load balancer cannot be deleted; you will see the error ``Invalid state PENDING_CREATE of loadbalancer resource``.
+To delete a load balancer in this state, you will need to manually update its provisioning status in the database.
+
+Find the database password:
+
+.. code-block:: console
+
+  ansible-vault view --vault-password-file <path-to-vault-pw> $KOLLA_CONFIG_PATH/passwords.yml
+
+  # Search for database_password with:
+  /^database
+
+Access the database from a controller:
+
+.. code-block:: console
+
+  docker exec -it mariadb bash
+  mysql -u root -p  octavia
+  # Enter the database password when promted.
+
+List the load balancers to find the ID of the broken one(s):
+
+.. code-block:: console
+
+  SELECT * FROM load_balancer;
+
+Set the provisioning status to ERROR for any broken load balancer:
+
+.. code-block:: console
+
+  UPDATE load_balancer SET provisioning_status='ERROR' WHERE id='<id>';
+
+Delete the load balancer from the OpenStack CLI, cascading if any stray
+Amphorae are hanging around:
+
+.. code-block:: console
+
+  openstack loadbalancer delete <id> --cascade
+
+
+Sometimes, Amphora may also fail to delete if they are stuck in state
+``BOOTING``. These can be resolved entirely from the OpenStack CLI:
+
+.. code-block:: console
+
+  openstack loadbalancer amphora configure <amphora-id>
+  openstack loadbalancer amphora delete <amphora-id>

From 46649f335b0e40fd2fcb068124d03a0d792d7fdb Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 9 Nov 2023 10:18:35 +0000
Subject: [PATCH 03/27] docs: a couple of bits for the RL9 migration

---
 doc/source/operations/rocky-linux-9.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst
index 9113f287a..c508d795c 100644
--- a/doc/source/operations/rocky-linux-9.rst
+++ b/doc/source/operations/rocky-linux-9.rst
@@ -98,6 +98,8 @@ suggestions:
 * Check for unexpected ``ERROR`` or ``CRITICAL`` messages in Kibana/OpenSearch
   Dashboard.
 * Check Grafana dashboards.
+* Disable Ansible fact caching for the duration of the migration, or remember
+  to clear hosts from the fact cache after they have been reprovisioned.
 
 Migrate to OpenSearch
 ---------------------
@@ -108,6 +110,7 @@ Elasticsearch/Kibana should be migrated to OpenSearch.
   docs <https://docs.openstack.org/kolla-ansible/yoga/reference/logging-and-monitoring/central-logging-guide-opensearch.html#migration>`__
 - If necessary, take a backup of the Elasticsearch data.
 - Ensure ``kolla_enable_elasticsearch`` is unset in ``etc/kayobe/kolla.yml``
+- If you have a custom Kolla Ansible inventory, ensure that it contains the ``opensearch`` and ``opensearch-dashboards`` groups. Otherwise, sync with the inventory in Kayobe.
 - Set ``kolla_enable_opensearch: true`` in ``etc/kayobe/kolla.yml``
 - ``kayobe overcloud service configuration generate --node-config-dir '/tmp/ignore' --kolla-tags none``
 - ``kayobe overcloud container image pull -kt opensearch``

From 5d2adaba9abf142e0550cf40852bf4eee81a2c3b Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 9 Nov 2023 12:37:30 +0000
Subject: [PATCH 04/27] OVN: Disable distributed floating IPs by default

It could be dangerous to enable distributed floating IPs on an OVN
system already deployed without them.
---
 etc/kayobe/kolla/globals.yml                        |  4 ----
 releasenotes/notes/enable_ovn-b84974f398242dcb.yaml | 12 +++++++-----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index c8f17d3b3..88eddd1bd 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -48,7 +48,3 @@ prometheus_ceph_mgr_exporter_endpoints:
 
 # Use inventory hostnames as labels
 prometheus_instance_label: "{% raw %}{{ ansible_facts.hostname }}{% endraw %}"
-
-#############################################################################
-
-neutron_ovn_distributed_fip: true
diff --git a/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml
index 21e253eda..3267a0447 100644
--- a/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml
+++ b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml
@@ -1,8 +1,10 @@
 ---
 upgrade:
   - |
-    Enabled ML2/OVN by default as checks preventing accidental migration
-    from ML2/OVS were added in kolla-ansible. If you are using a Neutron
-    plugin other than ML2/OVN, set `kolla_enable_ovn` to `false`.
-    OVN distributed FIP is also enabled, to disable it set
-    `neutron_ovn_distributed_fip` to `false` in `etc/kayobe/kolla/globals.yml`.
\ No newline at end of file
+    Enabled ML2/OVN by default. Checks preventing accidental migration
+    from ML2/OVS were added in Kolla Ansible. If you are using a Neutron
+    plugin other than ML2/OVN, set ``kolla_enable_ovn`` to ``false``.
+
+    OVN distributed FIP is disabled, to enable it set
+    ``neutron_ovn_distributed_fip`` to ``true`` in
+    ``etc/kayobe/kolla/globals.yml``.

From e3d662c3082d0513391e4088bb6b2d0b146431ab Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Thu, 9 Nov 2023 15:58:31 +0100
Subject: [PATCH 05/27] Update Consul and Vault to latest MPL releases

---
 etc/kayobe/inventory/group_vars/all/vault                     | 4 ++--
 .../notes/bump-hashicorp-images-7ef66e1328983bbe.yaml         | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml

diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault
index 3f004baff..dae698b54 100644
--- a/etc/kayobe/inventory/group_vars/all/vault
+++ b/etc/kayobe/inventory/group_vars/all/vault
@@ -10,13 +10,13 @@ seed_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stack
 seed_consul_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/consul"
 
 # Seed Consul container image tag.
-seed_consul_docker_tag: "1.16.1"
+seed_consul_docker_tag: "1.16.3"
 
 # Seed Vault container image.
 seed_vault_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/vault"
 
 # Seed Vault container image tag.
-seed_vault_docker_tag: "1.14.1"
+seed_vault_docker_tag: "1.14.6"
 
 # Seed Vault PKI Role name
 seed_vault_pki_role_name: "ServerCert"
diff --git a/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml
new file mode 100644
index 000000000..6e25cb43b
--- /dev/null
+++ b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml
@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    Updates Consul to 1.16.3 and Vault to 1.14.6.

From 60ec7a98428f50858ca3bbcda5c98d7e169a7b62 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Fri, 10 Nov 2023 09:31:56 +0100
Subject: [PATCH 06/27] Handle undefined Ansible variable

The variable kolla_enable_prometheus_ceph_mgr_exporter can be undefined,
leading to an error running the cephadm-commands-post.yml playbook.
---
 etc/kayobe/cephadm.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml
index 3f3cc261f..15460331b 100644
--- a/etc/kayobe/cephadm.yml
+++ b/etc/kayobe/cephadm.yml
@@ -92,7 +92,7 @@ cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post
 cephadm_commands_pre_default: []
 cephadm_commands_pre_extra: []
 
-cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | bool else [] }}"
+cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}"
 cephadm_commands_post_extra: []
 
 ###############################################################################

From 4620b80997cf44fc49f7afbda851caff09f1e87d Mon Sep 17 00:00:00 2001
From: technowhizz <7688823+technowhizz@users.noreply.github.com>
Date: Fri, 10 Nov 2023 10:22:41 +0000
Subject: [PATCH 07/27] Fix growroot playbook for disks ending with p

---
 etc/kayobe/ansible/growroot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml
index 54819b4c4..cdd7293d9 100644
--- a/etc/kayobe/ansible/growroot.yml
+++ b/etc/kayobe/ansible/growroot.yml
@@ -77,7 +77,7 @@
           vars:
             pv: "{{ pvs.stdout | from_json }}"
             disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}"
-            disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}"
+            disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' and disk_tmp[:4] == 'nvme' else disk_tmp }}"
             part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}"
           become: true
           failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout"

From 3e25167344668fab321a9b40321ab968537851ec Mon Sep 17 00:00:00 2001
From: technowhizz <7688823+technowhizz@users.noreply.github.com>
Date: Fri, 10 Nov 2023 10:26:19 +0000
Subject: [PATCH 08/27] Add release note

---
 .../notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml       | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml

diff --git a/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml
new file mode 100644
index 000000000..f3e4b84d2
--- /dev/null
+++ b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes an issue with the growroot playbook where disks such as 'sdp' would
+    become 'sd' due to the removal of the trailing 'p' when dealing with nvme
+    devices.

From f055b96ede6dbb828f66ec068e48e1194a18e740 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 10 Nov 2023 14:24:57 +0000
Subject: [PATCH 09/27] Build Neutron from StackHPC fork

---
 etc/kayobe/kolla.yml                                          | 4 ++++
 .../neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml  | 4 ++++
 2 files changed, 8 insertions(+)
 create mode 100644 releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index f4eeeedf3..3b8c73928 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -140,6 +140,10 @@ kolla_sources:
     type: git
     location: https://github.com/stackhpc/magnum.git
     reference: stackhpc/zed
+  neutron-base:
+    type: git
+    location: https://github.com/stackhpc/neutron.git
+    reference: stackhpc/{{ openstack_release }}
   neutron-base-plugin-networking-generic-switch:
     type: git
     location: https://github.com/stackhpc/networking-generic-switch.git
diff --git a/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml
new file mode 100644
index 000000000..cf0a68153
--- /dev/null
+++ b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Neutron containers are now built from our StackHPC fork.

From 9646d75183e24b0a9ae77cd50d584480bb5c0e2f Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 10 Nov 2023 15:28:55 +0000
Subject: [PATCH 10/27] Bump Neutron tag to fix multiple bugs

---
 etc/kayobe/kolla/globals.yml                             | 2 ++
 ...fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml | 6 ++++++
 ...ion-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml | 9 +++++++++
 3 files changed, 17 insertions(+)
 create mode 100644 releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml
 create mode 100644 releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 19dfa9e3d..1b0c14886 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -20,6 +20,8 @@ kayobe_image_tags:
     # with signature: `Missing section footer for 0000:00:01.3/piix4_pm`. Test carefully before bumping.
     centos: yoga-20230718T112646
 
+neutron_tag: yoga-20231110T152125
+
 nova_tag: yoga-20231103T161400
 nova_libvirt_tag: "{% raw %}{{ kayobe_image_tags['nova_libvirt'][kolla_base_distro] | default(nova_tag) }}{% endraw %}"
 
diff --git a/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml
new file mode 100644
index 000000000..dc5f3c2bf
--- /dev/null
+++ b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes Neutron so that load balancer FIPs are not broken on Neutron restart.
+    See `Neutron bug report
+    <https://bugs.launchpad.net/neutron/+bug/2042938>`__.
diff --git a/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml
new file mode 100644
index 000000000..bec2cafb8
--- /dev/null
+++ b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixes issue where Netmiko devices were sending no commands to the switch
+    since plug_bond_to_network is overridden in
+    networking_generic_switch/devices/netmiko_devices/init.py and
+    PLUG_BOND_TO_NETWORK to set to None.
+    See `NGS bug report
+    <https://bugs.launchpad.net/networking-generic-switch/+bug/2041516>`__.

From 16f33e2f26527ab82f6f232f6bd506058acd034f Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Fri, 10 Nov 2023 17:31:11 +0100
Subject: [PATCH 11/27] Bump Ceph images to latest stable releases

---
 etc/kayobe/cephadm.yml                                    | 2 +-
 releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml

diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml
index 15460331b..0e0b2fceb 100644
--- a/etc/kayobe/cephadm.yml
+++ b/etc/kayobe/cephadm.yml
@@ -12,7 +12,7 @@ cephadm_ceph_release: "{{ 'quincy' if (ansible_facts['distribution_release'] ==
 cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}"
 
 # Ceph container image tag.
-cephadm_image_tag: "{{ 'v17.2.6' if os_release == 'jammy' else 'v16.2.11' }}"
+cephadm_image_tag: "{{ 'v17.2.7' if os_release == 'jammy' else 'v16.2.14' }}"
 
 # Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy.
 cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}"
diff --git a/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml
new file mode 100644
index 000000000..9f51216a5
--- /dev/null
+++ b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml
@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    Updates default Ceph images to v17.2.7 for Quincy and v16.2.14 for Pacific.

From be9efd1c9f9c76fd352cbd6a7e877f8a155b9c07 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Mon, 13 Nov 2023 09:27:53 +0000
Subject: [PATCH 12/27] Check if repos backup exists in kolla footer

When a child service is build, such as neutron-infoblox-ipam-agent
being a child of neutron-server, the fotoer is ran twice.

This means it fails as the repo backup has already been moved.
---
 etc/kayobe/kolla.yml | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 17257688d..d16e6dd1c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -352,18 +352,22 @@ kolla_build_blocks:
     {% if stackhpc_kolla_clean_up_repo_mirrors | bool %}
     {% if kolla_base_distro in ['centos', 'rocky'] %}
     RUN \
-        tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
-        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
-           echo "Found repository mirror in Yum repositories"; \
-           exit 1; \
-        fi && \
-        rm -rf /etc/yum.repos.d.backup/
+        if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \
+          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
+          if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
+            echo "Found repository mirror in Yum repositories"; \
+            exit 1; \
+          fi && \
+          rm -rf /etc/yum.repos.d.backup/
+        fi
     {% else %}
     RUN \
-        mv /etc/apt/sources.list.backup /etc/apt/sources.list && \
-        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
-           echo "Found repository mirror in APT repositories"; \
-           exit 1; \
+        if [ -f /etc/apt/sources.list.backup ]; then \
+          mv /etc/apt/sources.list.backup /etc/apt/sources.list && \
+          if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
+            echo "Found repository mirror in APT repositories"; \
+            exit 1; \
+          fi
         fi
     {% endif %}
     {% endif %}

From eb3b4a95399830bf79aa61e856832cfff71864a0 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Mon, 13 Nov 2023 11:18:56 +0100
Subject: [PATCH 13/27] Restore flavor_id on openstack_nova_server_status
 metrics

The openstack-exporter version used in Zed started to use the latest
Nova API microversion, which produces an invalid flavor_id label on
openstack_nova_server_status metrics.

Force the use of the 2.1 API until we upgrade to a version of
openstack-exporter that can handle the latest API microversion.
---
 etc/kayobe/kolla/globals.yml                                 | 5 +++++
 .../notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml  | 5 +++++
 2 files changed, 10 insertions(+)
 create mode 100644 releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 88eddd1bd..46e19a1be 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -48,3 +48,8 @@ prometheus_ceph_mgr_exporter_endpoints:
 
 # Use inventory hostnames as labels
 prometheus_instance_label: "{% raw %}{{ ansible_facts.hostname }}{% endraw %}"
+
+# Make openstack-exporter use Nova API version 2.1 to keep metrics the same as
+# in Yoga. This is required to include a valid value for the flavor_id label on
+# openstack_nova_server_status metrics.
+prometheus_openstack_exporter_compute_api_version: "2.1"
diff --git a/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml
new file mode 100644
index 000000000..e4e2f43ac
--- /dev/null
+++ b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Restores valid value for the ``flavor_id`` label on
+    ``openstack_nova_server_status`` Prometheus metrics.

From 3be54580498c569c019dec9c987c11075f614dce Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Mon, 13 Nov 2023 10:27:41 +0000
Subject: [PATCH 14/27] Still check for stackhpc repos in footer

---
 etc/kayobe/kolla.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index d16e6dd1c..66d8171a1 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -353,21 +353,21 @@ kolla_build_blocks:
     {% if kolla_base_distro in ['centos', 'rocky'] %}
     RUN \
         if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \
-          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
-          if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
-            echo "Found repository mirror in Yum repositories"; \
-            exit 1; \
-          fi && \
+          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d
           rm -rf /etc/yum.repos.d.backup/
-        fi
+        fi && \
+        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
+          echo "Found repository mirror in Yum repositories"; \
+          exit 1; \
+        fi && \
     {% else %}
     RUN \
         if [ -f /etc/apt/sources.list.backup ]; then \
-          mv /etc/apt/sources.list.backup /etc/apt/sources.list && \
-          if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
-            echo "Found repository mirror in APT repositories"; \
-            exit 1; \
-          fi
+          mv /etc/apt/sources.list.backup /etc/apt/sources.list
+        fi && \
+        if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
+          echo "Found repository mirror in APT repositories"; \
+          exit 1; \
         fi
     {% endif %}
     {% endif %}

From 1f8962a7a47a1377ec2f6907f2c23f6abcd53dd2 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Mon, 13 Nov 2023 11:45:51 +0000
Subject: [PATCH 15/27] Apply suggestions from code review

Co-authored-by: Mark Goddard <mark@stackhpc.com>
---
 etc/kayobe/kolla.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 66d8171a1..fc93c1558 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -353,17 +353,17 @@ kolla_build_blocks:
     {% if kolla_base_distro in ['centos', 'rocky'] %}
     RUN \
         if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \
-          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d
-          rm -rf /etc/yum.repos.d.backup/
+          tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
+          rm -rf /etc/yum.repos.d.backup/ \
         fi && \
         if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
           echo "Found repository mirror in Yum repositories"; \
           exit 1; \
-        fi && \
+        fi
     {% else %}
     RUN \
         if [ -f /etc/apt/sources.list.backup ]; then \
-          mv /etc/apt/sources.list.backup /etc/apt/sources.list
+          mv /etc/apt/sources.list.backup /etc/apt/sources.list \
         fi && \
         if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
           echo "Found repository mirror in APT repositories"; \

From 7c2f630d8257b0fd0f453aa2f28729bdc219a61e Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Mon, 13 Nov 2023 12:34:38 +0000
Subject: [PATCH 16/27] Fix bash in footer

---
 etc/kayobe/kolla.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index fc93c1558..1f975c609 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -354,7 +354,7 @@ kolla_build_blocks:
     RUN \
         if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \
           tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
-          rm -rf /etc/yum.repos.d.backup/ \
+          rm -rf /etc/yum.repos.d.backup/; \
         fi && \
         if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \
           echo "Found repository mirror in Yum repositories"; \
@@ -363,7 +363,7 @@ kolla_build_blocks:
     {% else %}
     RUN \
         if [ -f /etc/apt/sources.list.backup ]; then \
-          mv /etc/apt/sources.list.backup /etc/apt/sources.list \
+          mv /etc/apt/sources.list.backup /etc/apt/sources.list; \
         fi && \
         if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \
           echo "Found repository mirror in APT repositories"; \

From 85204273fe554ef1a9a2231d6e9f2a9664b808f5 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Tue, 14 Nov 2023 09:26:30 +0100
Subject: [PATCH 17/27] Fix libvirtd memory leak in CentOS Stream 8

All nova images were rebuilt for CentOS Stream 8. Images for other
distributions are unmodified because they are not affected.

The qemu-kvm package is pinned to the latest version able to live
migrate from qemu-kvm-6.2.0-20.module_el8.7.0+1218+f626c2ff in the
stackhpc kolla fork [1].

[1] https://github.com/stackhpc/kolla/pull/279
---
 etc/kayobe/kolla/globals.yml                          | 11 +++++------
 .../fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml     |  6 ++++++
 2 files changed, 11 insertions(+), 6 deletions(-)
 create mode 100644 releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 51d2eb7df..feffb1dcb 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -15,14 +15,13 @@ kayobe_image_tags:
     centos: yoga-20231024T093507
     rocky: yoga-20231024T093507
     ubuntu: yoga-20231024T093507
-  nova_libvirt:
-    # Live-migration is broken from qemu-kvm-6.2.0-20.module_el8.7.0+1218+f626c2ff to qemu-kvm-6.2.0-39.module_el8+669+76cc32af
-    # with signature: `Missing section footer for 0000:00:01.3/piix4_pm`. Test carefully before bumping.
-    centos: yoga-20230718T112646
+  nova:
+    centos: yoga-20231113T171023
+    rocky: yoga-20231103T161400
+    ubuntu: yoga-20231103T161400
 
 cloudkitty_tag: yoga-20231107T165648
-nova_tag: yoga-20231103T161400
-nova_libvirt_tag: "{% raw %}{{ kayobe_image_tags['nova_libvirt'][kolla_base_distro] | default(nova_tag) }}{% endraw %}"
+nova_tag: "{% raw %}{{ kayobe_image_tags['nova'][kolla_base_distro] }}{% endraw %}"
 
 # These overrides are currently redundant, but are kept because it's not obvious that you need them if setting haproxy_tag
 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}"
diff --git a/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml b/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml
new file mode 100644
index 000000000..30e9b5892
--- /dev/null
+++ b/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes a libvirtd memory leak on CentOS Stream 8. See `Red Hat Bugzilla bug
+    2143235 <https://bugzilla.redhat.com/show_bug.cgi?id=2143235>`__ for more
+    details.

From df4a45bb34be8fb3f3b409deead72530e8868d65 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 13 Nov 2023 16:34:22 +0000
Subject: [PATCH 18/27] Automatic openstack release for kolla sources

The NOTE(upgrade) tag has been added to more places to make it easier
to update the config for new releases
---
 .github/workflows/stackhpc-promote.yml |  2 +-
 etc/kayobe/kolla.yml                   | 17 +++++++----------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml
index 885ed0258..e5c254381 100644
--- a/.github/workflows/stackhpc-promote.yml
+++ b/.github/workflows/stackhpc-promote.yml
@@ -3,7 +3,7 @@ name: Promote package repositories
 on:
   push:
     branches:
-      # NOTE(mgoddard): Reference only the current release branch here.
+      # NOTE(upgrade): Reference only the current release branch here.
       - stackhpc/zed
 jobs:
   promote:
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 3b8c73928..35ba5523e 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -111,12 +111,9 @@ kolla_tag: "{{ openstack_release }}-{{ kolla_base_distro }}-{{ kolla_base_distro
 #     type: git
 #     location: https://github.com/openstack/ironic
 #     reference: master
-# NOTE (Alex-Welsh): The reference for many of these entries could be
-# 'stackhpc/{{ openstack_release }}' which would keep the branch up to date
-# for the current release. This is nice in theory but in practice, the stackhpc
-# forks change with every release and the elements in this list change with
-# them. Explicitly using /zed makes it more intuitive to find and edit these
-# entries.
+# NOTE(upgrade): These sources should be checked with each release. StackHPC
+# branches are only required when we have custom backports. For a new release,
+# we may have caught up with upstream.
 kolla_sources:
   bifrost-base-additions-stackhpc-inspector-plugins:
     # Install our custom inspector plugins.
@@ -126,11 +123,11 @@ kolla_sources:
   cloudkitty-base:
     type: git
     location: https://github.com/stackhpc/cloudkitty.git
-    reference: stackhpc/zed
+    reference: stackhpc/{{ openstack_release }}
   horizon-plugin-cloudkitty-dashboard:
     type: git
     location: https://github.com/stackhpc/cloudkitty-dashboard.git
-    reference: stackhpc/zed
+    reference: stackhpc/{{ openstack_release }}
   ironic-inspector-additions-stackhpc-inspector-plugins:
     # Install our custom inspector plugins.
     type: git
@@ -139,7 +136,7 @@ kolla_sources:
   magnum-base:
     type: git
     location: https://github.com/stackhpc/magnum.git
-    reference: stackhpc/zed
+    reference: stackhpc/{{ openstack_release }}
   neutron-base:
     type: git
     location: https://github.com/stackhpc/neutron.git
@@ -147,7 +144,7 @@ kolla_sources:
   neutron-base-plugin-networking-generic-switch:
     type: git
     location: https://github.com/stackhpc/networking-generic-switch.git
-    reference: stackhpc/zed
+    reference: stackhpc/{{ openstack_release }}
 
 ###############################################################################
 # Kolla image build configuration.

From 27a6469fbaddde0ccbfd2faa6b0677baf04ea4a8 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 9 Nov 2023 12:18:06 +0000
Subject: [PATCH 19/27] docs: add more notable changes and known issues for Zed

---
 doc/source/operations/upgrading.rst | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst
index 10827fcb3..7c2279058 100644
--- a/doc/source/operations/upgrading.rst
+++ b/doc/source/operations/upgrading.rst
@@ -71,6 +71,31 @@ to::
 
     ark.stackhpc.com/stackhpc/etcd:zed-rocky-9-20230821T155947
 
+OVN enabled by default
+----------------------
+
+OVN is now enabled by default in StackHPC Kayobe Configuration.  This change
+was made to align with our standard deployment configuration.
+
+There is currently not a tested migration path from OVS to OVN on a running
+system. If you are using a Neutron plugin other than ML2/OVN, set
+``kolla_enable_ovn`` to ``false`` in ``etc/kayobe/kolla.yml``.
+
+For new deployments using OVN, See
+:kolla-ansible-doc:`reference/networking/neutron.html#ovn-ml2-ovn`.
+
+Known issues
+============
+
+* Rebuilds of servers with volumes are broken if there are any Nova compute
+  services running an older release, including any that are down. Old compute
+  services should be removed using ``openstack compute service delete``, then
+  remaining compute services restarted. See `LP#2040264
+  <https://bugs.launchpad.net/nova/+bug/2040264>`__.
+
+* The OVN sync repair tool removes metadata ports, breaking OVN load balancers.
+  See `LP#2038091 <https://bugs.launchpad.net/neutron/+bug/2038091>`__.
+
 Security baseline
 =================
 

From 47dc799f2bbb1298c1f33f1283dba50dc1e0c0f5 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Tue, 14 Nov 2023 13:41:35 +0000
Subject: [PATCH 20/27] Add cloudkitty tag

---
 etc/kayobe/kolla/globals.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 88eddd1bd..f136474c6 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -20,10 +20,14 @@ kayobe_image_tags:
   ovn:
     rocky: zed-rocky-9-20230925T132313
     ubuntu: zed-ubuntu-jammy-20230821T155947
+  cloudkitty:
+    rocky: zed-rocky-9-20231114T124701
+    ubuntu: zed-ubuntu-jammy-20231114T124701
 
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"
 ovn_tag: "{% raw %}{{ kayobe_image_tags['ovn'][kolla_base_distro] }}{% endraw %}"
+cliudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
 
 om_enable_rabbitmq_high_availability: true
 

From e446d94b869986903c6c5b734dafa053c1200806 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Tue, 14 Nov 2023 13:57:00 +0000
Subject: [PATCH 21/27] New neutron tag

---
 etc/kayobe/kolla/globals.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index feffb1dcb..337687d65 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -22,6 +22,7 @@ kayobe_image_tags:
 
 cloudkitty_tag: yoga-20231107T165648
 nova_tag: "{% raw %}{{ kayobe_image_tags['nova'][kolla_base_distro] }}{% endraw %}"
+neutron_tag: yoga-20231114T125927
 
 # These overrides are currently redundant, but are kept because it's not obvious that you need them if setting haproxy_tag
 glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}"

From 3b1292536d2c45d27d0575e57cf60457b7379cb7 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 14 Nov 2023 14:06:29 +0000
Subject: [PATCH 22/27] docs: Fix typo

---
 doc/source/operations/upgrading.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst
index 7c2279058..94fc6000a 100644
--- a/doc/source/operations/upgrading.rst
+++ b/doc/source/operations/upgrading.rst
@@ -81,7 +81,7 @@ There is currently not a tested migration path from OVS to OVN on a running
 system. If you are using a Neutron plugin other than ML2/OVN, set
 ``kolla_enable_ovn`` to ``false`` in ``etc/kayobe/kolla.yml``.
 
-For new deployments using OVN, See
+For new deployments using OVN, see
 :kolla-ansible-doc:`reference/networking/neutron.html#ovn-ml2-ovn`.
 
 Known issues

From 3a36c8f0b2060eb413a96dbb8ce4460e0995944f Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 14 Nov 2023 20:41:25 +0000
Subject: [PATCH 23/27] Fix cloudkitty_tag

---
 etc/kayobe/kolla/globals.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 65df749bb..8880e5126 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -27,7 +27,7 @@ kayobe_image_tags:
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"
 ovn_tag: "{% raw %}{{ kayobe_image_tags['ovn'][kolla_base_distro] }}{% endraw %}"
-cliudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
+cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
 
 om_enable_rabbitmq_high_availability: true
 

From 405c6d4d28037ccf26cbb72ad39920fa7c77746e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 14 Nov 2023 20:42:34 +0000
Subject: [PATCH 24/27] Remove invalid reno for zed

---
 .../notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml     | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml

diff --git a/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml b/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml
deleted file mode 100644
index 30e9b5892..000000000
--- a/releasenotes/notes/fix-libvirt-memory-leak-1a8c7a66679b9de7.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-fixes:
-  - |
-    Fixes a libvirtd memory leak on CentOS Stream 8. See `Red Hat Bugzilla bug
-    2143235 <https://bugzilla.redhat.com/show_bug.cgi?id=2143235>`__ for more
-    details.

From ae93adfe8b19834c64f73b82298a6677627e1d4b Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Wed, 15 Nov 2023 10:38:01 +0000
Subject: [PATCH 25/27] Add Neutron tag

---
 etc/kayobe/kolla/globals.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 8880e5126..83e919339 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -23,11 +23,15 @@ kayobe_image_tags:
   cloudkitty:
     rocky: zed-rocky-9-20231114T124701
     ubuntu: zed-ubuntu-jammy-20231114T124701
+  neutron:
+    rocky: zed-rocky-9-20231115T094053
+    ubuntu: zed-ubuntu-jammy-20231115T094053
 
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"
 ovn_tag: "{% raw %}{{ kayobe_image_tags['ovn'][kolla_base_distro] }}{% endraw %}"
 cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}"
+neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}"
 
 om_enable_rabbitmq_high_availability: true
 

From 0c231402a748e565a8754242992f2ebda2c2d86b Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 15 Nov 2023 10:41:05 +0000
Subject: [PATCH 26/27] Fix Ceph reno for Zed

---
 releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml
index 9f51216a5..439f8b842 100644
--- a/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml
+++ b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml
@@ -1,4 +1,4 @@
 ---
 upgrade:
   - |
-    Updates default Ceph images to v17.2.7 for Quincy and v16.2.14 for Pacific.
+    Updates default Ceph images to v17.2.7 for Quincy.

From c426d5d724931929896af89299a338aaf04bd88b Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 16 Nov 2023 10:41:36 +0000
Subject: [PATCH 27/27] Update cloudkitty and neutron tags for 2023.1

---
 etc/kayobe/kolla/globals.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 6c7844c10..024efe4c8 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -13,11 +13,11 @@ kayobe_image_tags:
     rocky: 2023.1-rocky-9-20231013T151957
     ubuntu: 2023.1-ubuntu-jammy-20231013T151957
   cloudkitty:
-    rocky: TODO
-    ubuntu: TODO
+    rocky: 2023.1-rocky-9-20231115T110235
+    ubuntu: 2023.1-ubuntu-jammy-20231115T110235
   neutron:
-    rocky: TODO
-    ubuntu: TODO
+    rocky: 2023.1-rocky-9-20231115T110249
+    ubuntu: 2023.1-ubuntu-jammy-20231115T110249
 
 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}"
 bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}"