From 6fc586733145cdeb76f2e1fbd2f3c0a61cc6f548 Mon Sep 17 00:00:00 2001
From: Dawud M <7688823+technowhizz@users.noreply.github.com>
Date: Tue, 21 Mar 2023 19:09:14 +0000
Subject: [PATCH 01/14] Fix smartmon script

Fixes the smartmon script to detect SMART info values regardless of
their case. Previously, this would lead to wrongly formatted .prom files
resulting in node_exporter failing to parse them.
---
 etc/kayobe/ansible/scripts/smartmon.sh                   | 3 ++-
 .../notes/fix-smartmon-script-db2a38df1b245e58.yaml      | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml

diff --git a/etc/kayobe/ansible/scripts/smartmon.sh b/etc/kayobe/ansible/scripts/smartmon.sh
index 32b3ea81c..c08c46e60 100644
--- a/etc/kayobe/ansible/scripts/smartmon.sh
+++ b/etc/kayobe/ansible/scripts/smartmon.sh
@@ -106,6 +106,7 @@ parse_smartctl_scsi_attributes() {
 }
 
 parse_smartctl_info() {
+  shopt -s nocasematch
   local -i smart_available=0 smart_enabled=0 smart_healthy=
   local disk="$1" disk_type="$2"
   local model_family='' device_model='' serial_number='' fw_version='' vendor='' product='' revision='' lun_id=''
@@ -199,4 +200,4 @@ for device in ${device_list}; do
     exit
     ;;
   esac
-done | format_output
\ No newline at end of file
+done | format_output
diff --git a/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml
new file mode 100644
index 000000000..b71a5c439
--- /dev/null
+++ b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixes the smartmon script to be case insensitive when checking for
+    the inital SMART info. This is to ensure that the script works
+    correctly on systems where the output of `smartctl -i` is not
+    capitalised as previously expected by the script. This leads to
+    badly formatted .prom files which lead to node_exporter failing to
+    scrape the file.

From 1a61db89659749787c7f2a004240cfbf1423704f Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Tue, 28 Mar 2023 12:15:50 +0100
Subject: [PATCH 02/14] Add support for manila in multinode

Supports Manila in the ci-multinode environment using the CephFS native
backend. Configuration is automatic based on the Kolla variables to
enable manila and the backend.

Also moves cephadm-commands to run after other tasks so that the
commands can reference objects created earlier on.
---
 etc/kayobe/ansible/cephadm.yml                |  2 +-
 .../environments/ci-multinode/cephadm.yml     | 17 +++++++++++++++++
 .../group_vars/seed/network-interfaces        |  4 ++++
 .../environments/ci-multinode/kolla.yml       |  4 ++++
 .../ci-multinode/kolla/globals.yml            |  3 +++
 etc/kayobe/environments/ci-multinode/seed.yml | 19 ++++++++++++++++++-
 ...manila-for-multinode-df2186276c6b434a.yaml |  8 ++++++++
 7 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml

diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml
index 1cb4d56e9..de5f9fac4 100644
--- a/etc/kayobe/ansible/cephadm.yml
+++ b/etc/kayobe/ansible/cephadm.yml
@@ -1,8 +1,8 @@
 ---
 # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys.
 - import_playbook: cephadm-deploy.yml
-- import_playbook: cephadm-commands.yml
 - import_playbook: cephadm-ec-profiles.yml
 - import_playbook: cephadm-crush-rules.yml
 - import_playbook: cephadm-pools.yml
 - import_playbook: cephadm-keys.yml
+- import_playbook: cephadm-commands.yml
diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml
index addb37ddf..1c7a0bdf6 100644
--- a/etc/kayobe/environments/ci-multinode/cephadm.yml
+++ b/etc/kayobe/environments/ci-multinode/cephadm.yml
@@ -35,6 +35,12 @@ cephadm_pools:
   - name: vms
     application: rbd
     state: present
+  - name: cephfs_data
+    application: cephfs
+    state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}"
+  - name: cephfs_metadata
+    application: cephfs
+    state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}"
 
 # List of Cephx keys. See stackhpc.cephadm.keys role for format.
 cephadm_keys:
@@ -56,3 +62,14 @@ cephadm_keys:
       osd: "profile rbd pool=images"
       mgr: "profile rbd pool=images"
     state: present
+  - name: client.manila
+    caps:
+      mon: "allow r"
+      mgr: "allow rw"
+    state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}"
+
+# List of Cephadm commands to run. See stackhpc.cephadm.commands role for format.
+cephadm_commands: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}"
+cephadm_commands_manila_cephfs_native:
+  - "fs new manila-cephfs cephfs_metadata cephfs_data"
+  - "orch apply mds manila-cephfs"
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
index 0c868b110..a918946eb 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
@@ -10,6 +10,10 @@ external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}"
 
 public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}"
 
+# The storage interface is required for routing manila traffic between VMs and
+# the Storage nodes.
+storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml
index 5c6c80dae..a087ad66f 100644
--- a/etc/kayobe/environments/ci-multinode/kolla.yml
+++ b/etc/kayobe/environments/ci-multinode/kolla.yml
@@ -4,3 +4,7 @@ kolla_enable_cinder_backup: true
 kolla_enable_neutron_provider_networks: true
 kolla_enable_ovn: true
 kolla_enable_octavia: true
+
+# The multinode environment supports Manila but it is not enabled by default.
+# kolla_enable_manila: true
+# kolla_enable_manila_backend_cephfs_native: true
diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
index bd726ddc5..e0766e7fe 100644
--- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml
+++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
@@ -22,3 +22,6 @@ es_heap_size: 1g
 octavia_auto_configure: "no"
 octavia_provider_drivers: "ovn:OVN provider"
 octavia_provider_agents: "ovn"
+
+# Manila CephFS configuration
+manila_cephfs_filesystem_name: manila-cephfs
diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml
index 3eb7afa90..749c398d1 100644
--- a/etc/kayobe/environments/ci-multinode/seed.yml
+++ b/etc/kayobe/environments/ci-multinode/seed.yml
@@ -3,12 +3,29 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl
 seed_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"
 
+seed_extra_network_interfaces: >
+  "{{ seed_extra_network_interfaces_external +
+  (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}"
+
 # Seed has been provided an external interface
 # for tempest tests and SSH access to machines.
-seed_extra_network_interfaces:
+seed_extra_network_interfaces_external:
   - "external"
   - "public"
 
+# Seed requires access to the storage network for manila-cephfs.
+seed_extra_network_interfaces_manila:
+  - "storage"
+
 # Enable IP routing and source NAT on the seed, allowing it to be used as the
 # external subnet gateway and provide internet access for VMs in the deployment.
 seed_enable_snat: true
+
+snat_rules_default:
+  - interface: "{{ ansible_facts.default_ipv4.interface }}"
+    source_ip: "{{ ansible_facts.default_ipv4.address }}"
+snat_rules_manila:
+  - interface: "{{ storage_interface }}"
+    source_ip: "{{ ansible_facts[storage_interface].ipv4.address }}"
+# Only add the storage snat rule if we are using manila-cephfs.
+snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else snat_rules_default }}"
diff --git a/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml
new file mode 100644
index 000000000..6952fe9bc
--- /dev/null
+++ b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds support for Manila in the ci-multinode environment using the CephFS
+    native backend. This is disabled by default, but can be enabled by setting
+    the following variables in the kayobe configuration:
+    `kolla_enable_manila: true`
+    `kolla_enable_manila_backend_cephfs_native: true`

From 37d5671bf3abc08afe0bab78ddff1d2f9f386994 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Wed, 29 Mar 2023 15:09:57 +0100
Subject: [PATCH 03/14] Add documentation for multinode manila

---
 .../contributor/environments/ci-multinode.rst | 150 ++++++++++++++++++
 1 file changed, 150 insertions(+)

diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst
index 0dd787140..e256281d4 100644
--- a/doc/source/contributor/environments/ci-multinode.rst
+++ b/doc/source/contributor/environments/ci-multinode.rst
@@ -53,3 +53,153 @@ Final steps
 1. ``source kayobe-env --environment ci-aio``
 2. Run ``kayobe overcloud host configure``
 3. Run ``kayobe overcloud service deploy``
+
+
+Manila
+======
+The Multinode environment supports Manila with the CephFS native backend, but it
+is not enabled by default. To enable it, set the following in
+``/etc/kayobe/environments/ci-multinode/kolla.yml``:
+
+.. code-block:: yaml
+
+      kolla_enable_manila: true
+      kolla_enable_manila_backend_cephfs: true
+
+And re-run ``kayobe overcloud service deploy`` if you are working on an existing
+deployment.
+
+To test it, you will need two virtual machines. Cirros does not support the Ceph
+kernel client, so you will need to use a different image. Any regular Linux 
+distribution should work. As an example, we will use Ubuntu 20.04.
+
+Download the image locally:
+
+.. code-block:: bash
+
+      wget http://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
+
+Upload the image to Glance:
+
+.. code-block:: bash
+
+      openstack image create --container-format bare --disk-format qcow2 --file focal-server-cloudimg-amd64.img Ubuntu-20.04 --progress
+
+Create a keypair:
+
+.. code-block:: bash
+
+      openstack keypair create --private-key ~/.ssh/id_rsa id_rsa
+
+Create two virtual machines from the image:
+
+.. code-block:: bash
+
+      openstack server create --flavor m1.small --image Ubuntu-20.04 --key-name id_rsa --network admin-tenant ubuntu-client-1
+      openstack server create --flavor m1.small --image Ubuntu-20.04 --key-name id_rsa --network admin-tenant ubuntu-client-2
+
+Wait until the instances are active. It is worth noting that this process can
+take a while, especially if the overcloud is deployed to virtual machines. You
+can monitor the progress with the following command:
+
+.. code-block:: bash
+
+      watch openstack server list
+
+Once they are active, create two floating IPs:
+
+.. code-block:: bash
+
+      openstack floating ip create external
+      openstack floating ip create external
+
+Associate the floating IPs to the instances:
+
+.. code-block:: bash
+
+      openstack server add floating ip ubuntu-client-1 <floating-ip-1>
+      openstack server add floating ip ubuntu-client-2 <floating-ip-2>
+
+
+Then SSH into each instance and install the Ceph client:
+
+.. code-block:: bash
+   
+      sudo apt update
+      sudo apt install -y ceph-common
+
+
+Back on the host, install the Manila client:
+
+.. code-block:: bash
+
+      sudo dnf install -y python-manilaclient
+
+Then create a share type and share:
+
+.. code-block:: bash
+
+      manila type-create cephfs-type false --is_public true
+      manila type-key cephfs-type set vendor_name=Ceph storage_protocol=CEPHFS
+      manila create --name test-share --share-type cephfs-type CephFS 2
+
+Wait until the share is available:
+
+.. code-block:: bash
+
+      manila list
+
+Then allow access to the shares to two users:
+
+.. code-block:: bash
+
+      manila access-allow test-share cephx alice
+      manila access-allow test-share cephx bob
+
+Show the access list to make sure the state of both entries is ``active`` and
+take note of the access keys:
+
+.. code-block:: bash
+
+      manila access-list test-share
+
+And take note of the path to the share:
+
+.. code-block:: bash
+
+      manila share-export-location-list test-share
+
+SSH into the first instance, create a directory for the share, and mount it:
+
+.. code-block:: bash
+
+      mkdir testdir
+      sudo mount -t ceph {path} -o name=alice,secret='{access_key}' testdir
+
+Where the path is the path to the share from the previous step, and the secret
+is the access key for the user alice.
+
+Then create a file in the share:
+
+.. code-block:: bash
+
+      sudo touch testdir/testfile
+
+SSH into the second instance, create a directory for the share, and mount it:
+
+.. code-block:: bash
+
+      mkdir testdir
+      sudo mount -t ceph {path} -o name=bob,secret='{access_key}' testdir
+
+Where the path is the same as before, and the secret is the access key for the
+user bob.
+
+Then check that the file created in the first instance is visible in the second
+instance:
+
+.. code-block:: bash
+
+      ls testdir
+
+If it shows the test file then the share is working correctly.

From 2aa47037cd9540596f1f394da8d20d008a24cf36 Mon Sep 17 00:00:00 2001
From: Dawud M <7688823+technowhizz@users.noreply.github.com>
Date: Tue, 21 Mar 2023 18:36:19 +0000
Subject: [PATCH 04/14] Fix temperature graph on hardware overview dash

Fixes the hardware overview dashboard to use the correct metric for
displaying drive temps. Now uses an `or` to display whichever metric is
compatible with the drives in the system. The two metrics are
temperature_case_raw_value and temperature_celsius_raw_value.
---
 .../grafana/dashboards/openstack/hardware_overview.json    | 6 +++---
 ...-hardware-overview-dashboard-temp-ecc047ba46305668.yaml | 7 +++++++
 2 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml

diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json
index 63bbf4af3..60649ff28 100644
--- a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json
+++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json
@@ -530,7 +530,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"}",
+          "expr": "smartmon_temperature_case_raw_value{instance=~\"$node\"} or smartmon_temperature_celsius_raw_value{instance=~\"$node\"}",
           "format": "table",
           "hide": false,
           "instant": true,
@@ -664,7 +664,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h])",
+          "expr": "avg_over_time(smartmon_temperature_case_raw_value{instance=~\"$node\"}[1h]) or avg_over_time(smartmon_temperature_celsius_raw_value{instance=~\"$node\"}[1h])",
           "instant": false,
           "interval": "",
           "legendFormat": "{{instance}} - {{disk}} - {{serial_number}}",
@@ -752,4 +752,4 @@
   "version": 5,
   "weekStart": ""
 }
-{% endraw %}
\ No newline at end of file
+{% endraw %}
diff --git a/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml
new file mode 100644
index 000000000..6a395888b
--- /dev/null
+++ b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml
@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    Fixes the hardware overview dashboard to use the correct metric for
+    displaying drive temps. Now uses an `or` to display whichever metric
+    is compatible with the drives in the system. The two metrics are
+    temperature_case_raw_value and temperature_celsius_raw_value.

From 92a30f0ad1ac8e69e9a1cb1f77e2e032b2c02082 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 30 Mar 2023 10:16:47 +0100
Subject: [PATCH 05/14] Split cephadm_commands into pre and post

Split `cephadm_commands` into `cephadm_commands_pre` and
`cephadm_commands_post` commands. This allows the user
to run commands that must be executed before the rest of
the post-deployment configuration, as well as commands
that rely on resources created by the post-deployment config.
---
 doc/source/configuration/cephadm.rst              | 15 +++++++++++----
 .../contributor/environments/ci-multinode.rst     |  4 ++--
 ...adm-commands.yml => cephadm-commands-post.yml} |  4 +++-
 etc/kayobe/ansible/cephadm-commands-pre.yml       | 13 +++++++++++++
 etc/kayobe/ansible/cephadm.yml                    |  3 ++-
 etc/kayobe/cephadm.yml                            |  9 ++++++---
 etc/kayobe/environments/ci-multinode/cephadm.yml  |  4 +++-
 ...ephadm-commands-pre-post-bddd25b8bd171ab0.yaml |  8 ++++++++
 8 files changed, 48 insertions(+), 12 deletions(-)
 rename etc/kayobe/ansible/{cephadm-commands.yml => cephadm-commands-post.yml} (52%)
 create mode 100644 etc/kayobe/ansible/cephadm-commands-pre.yml
 create mode 100644 releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml

diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst
index 0322482c0..dfab2b748 100644
--- a/doc/source/configuration/cephadm.rst
+++ b/doc/source/configuration/cephadm.rst
@@ -242,19 +242,26 @@ for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible.
 Ceph Commands
 ~~~~~~~~~~~~~
 
-It is possible to run an arbitrary list of commands against the cluster after deployment
-by setting the ``cephadm_commands`` variable. ``cephadm_commands`` should be a list of commands
-to pass to ``cephadm shell -- ceph``. For example:
+It is possible to run an arbitrary list of commands against the cluster after
+deployment by setting the ``cephadm_commands_pre`` and ``cephadm_commands_post``
+variables. Each should be a list of commands to pass to ``cephadm shell --
+ceph``. For example:
 
 .. code:: yaml
 
    # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands
    # for format.
-   cephadm_commands:
+   cephadm_commands_pre:
     # Configure Prometheus exporter to listen on a specific interface. The default
     # is to listen on all interfaces.
     - "config set mgr mgr/prometheus/server_addr 10.0.0.1"
 
+Both variables have the same format, however commands in the
+``cephadm_commands_pre`` list are executed before the rest of the Ceph
+post-deployment configuration is applied. Commands in the
+``cephadm_commands_post`` list are executed after the rest of the Ceph
+post-deployment configuration is applied.
+
 Deployment
 ==========
 
diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst
index e256281d4..e42d61807 100644
--- a/doc/source/contributor/environments/ci-multinode.rst
+++ b/doc/source/contributor/environments/ci-multinode.rst
@@ -70,7 +70,7 @@ And re-run ``kayobe overcloud service deploy`` if you are working on an existing
 deployment.
 
 To test it, you will need two virtual machines. Cirros does not support the Ceph
-kernel client, so you will need to use a different image. Any regular Linux 
+kernel client, so you will need to use a different image. Any regular Linux
 distribution should work. As an example, we will use Ubuntu 20.04.
 
 Download the image locally:
@@ -124,7 +124,7 @@ Associate the floating IPs to the instances:
 Then SSH into each instance and install the Ceph client:
 
 .. code-block:: bash
-   
+
       sudo apt update
       sudo apt install -y ceph-common
 
diff --git a/etc/kayobe/ansible/cephadm-commands.yml b/etc/kayobe/ansible/cephadm-commands-post.yml
similarity index 52%
rename from etc/kayobe/ansible/cephadm-commands.yml
rename to etc/kayobe/ansible/cephadm-commands-post.yml
index 32742a343..fbca30dfd 100644
--- a/etc/kayobe/ansible/cephadm-commands.yml
+++ b/etc/kayobe/ansible/cephadm-commands-post.yml
@@ -1,5 +1,5 @@
 ---
-- name: Ensure additional Ceph commands are run
+- name: Ensure additional Ceph commands are run after post-deployment configuration
   gather_facts: false
   hosts: mons
   become: true
@@ -9,3 +9,5 @@
   tasks:
     - import_role:
         name: stackhpc.cephadm.commands
+      vars:
+        cephadm_commands: "{{ cephadm_commands_post }}"
diff --git a/etc/kayobe/ansible/cephadm-commands-pre.yml b/etc/kayobe/ansible/cephadm-commands-pre.yml
new file mode 100644
index 000000000..08047cc51
--- /dev/null
+++ b/etc/kayobe/ansible/cephadm-commands-pre.yml
@@ -0,0 +1,13 @@
+---
+- name: Ensure additional Ceph commands are run before post-deployment configuration
+  gather_facts: false
+  hosts: mons
+  become: true
+  tags:
+    - cephadm
+    - cephadm-commands
+  tasks:
+    - import_role:
+        name: stackhpc.cephadm.commands
+      vars:
+        cephadm_commands: "{{ cephadm_commands_pre }}"
diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml
index de5f9fac4..b699df153 100644
--- a/etc/kayobe/ansible/cephadm.yml
+++ b/etc/kayobe/ansible/cephadm.yml
@@ -1,8 +1,9 @@
 ---
 # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys.
 - import_playbook: cephadm-deploy.yml
+- import_playbook: cephadm-commands-pre.yml
 - import_playbook: cephadm-ec-profiles.yml
 - import_playbook: cephadm-crush-rules.yml
 - import_playbook: cephadm-pools.yml
 - import_playbook: cephadm-keys.yml
-- import_playbook: cephadm-commands.yml
+- import_playbook: cephadm-commands-post.yml
diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml
index 4f5f0ed7c..ae06880ca 100644
--- a/etc/kayobe/cephadm.yml
+++ b/etc/kayobe/cephadm.yml
@@ -60,9 +60,12 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}"
 # List of Cephx keys. See stackhpc.cephadm.keys role for format.
 #cephadm_keys:
 
-# A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands
-# for format.
-#cephadm_commands:
+# A list of commands to pass to cephadm shell -- ceph. See
+# stackhpc.cephadm.commands for format. Pre commands run before the rest of the
+# post-deployment configuration, post commands run after the rest of the
+# post-deployment configuration.
+#cephadm_commands_pre:
+#cephadm_commands_post:
 
 ###############################################################################
 # Kolla Ceph auto-configuration.
diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml
index 1c7a0bdf6..7885a5735 100644
--- a/etc/kayobe/environments/ci-multinode/cephadm.yml
+++ b/etc/kayobe/environments/ci-multinode/cephadm.yml
@@ -69,7 +69,9 @@ cephadm_keys:
     state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}"
 
 # List of Cephadm commands to run. See stackhpc.cephadm.commands role for format.
-cephadm_commands: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}"
+cephadm_commands_pre: []
+
+cephadm_commands_post: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}"
 cephadm_commands_manila_cephfs_native:
   - "fs new manila-cephfs cephfs_metadata cephfs_data"
   - "orch apply mds manila-cephfs"
diff --git a/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml
new file mode 100644
index 000000000..c39ffa43a
--- /dev/null
+++ b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Split `cephadm_commands` into `cephadm_commands_pre` and
+    `cephadm_commands_post` commands. This allows the user to run commands that
+    must be run before the rest of the post-deployment configuration, as well
+    as commands that rely on resources created by the post-deployment config.
+

From ff6abe95192cb97d9cb20eac9e6b9dc3b21eaf57 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Thu, 30 Mar 2023 11:43:46 +0100
Subject: [PATCH 06/14] Update documentation for Manila and Ceph

---
 doc/source/configuration/cephadm.rst          | 44 +++++++++++++++++++
 .../contributor/environments/ci-multinode.rst | 10 ++---
 2 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst
index dfab2b748..8908b351f 100644
--- a/doc/source/configuration/cephadm.rst
+++ b/doc/source/configuration/cephadm.rst
@@ -262,6 +262,50 @@ post-deployment configuration is applied. Commands in the
 ``cephadm_commands_post`` list are executed after the rest of the Ceph
 post-deployment configuration is applied.
 
+Manila & CephFS
+~~~~~~~~~~~~~~~
+
+Using Manila with the CephFS backend requires the configuration of additional
+resources.
+
+A Manila key should be added to cephadm_keys:
+
+.. code:: yaml
+
+  # Append the following to cephadm_keys:
+  - name: client.manila
+    caps:
+      mon: "allow r"
+      mgr: "allow rw"
+    state: present
+
+A CephFS filesystem requires two pools, one for metadata and one for data:
+
+.. code:: yaml
+
+  # Append the following to cephadm_pools:
+  - name: cephfs_data
+    application: cephfs
+    state: present
+  - name: cephfs_metadata
+    application: cephfs
+    state: present
+
+Finally, the CephFS filesystem itself should be created:
+
+.. code:: yaml
+
+  # Append the following to cephadm_commands_post:
+  - "fs new manila-cephfs cephfs_metadata cephfs_data"
+  - "orch apply mds manila-cephfs"
+
+In this example, the filesystem is named ``manila-cephfs``. This name
+should be used in the Kolla Manila configuration e.g.:
+
+.. code:: yaml
+
+  manila_cephfs_filesystem_name: manila-cephfs
+
 Deployment
 ==========
 
diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst
index e42d61807..93341113a 100644
--- a/doc/source/contributor/environments/ci-multinode.rst
+++ b/doc/source/contributor/environments/ci-multinode.rst
@@ -38,11 +38,11 @@ Config changes
 3. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000
 4. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes
    conflicts, change to 4790)
-5. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so
+5. In etc/kayobe/environments/ci-multinode/tf-networks.yml, edit admin_ips so
    that the compute and controller IPs line up with the
    instances that were created earlier, remove the other IPs for seed and
    cephOSD
-6. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all
+6. In etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all
    the entries and just assign ``aio_ips:`` an empty set ``[]``
 7. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed
 8. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails,
@@ -59,7 +59,7 @@ Manila
 ======
 The Multinode environment supports Manila with the CephFS native backend, but it
 is not enabled by default. To enable it, set the following in
-``/etc/kayobe/environments/ci-multinode/kolla.yml``:
+``etc/kayobe/environments/ci-multinode/kolla.yml``:
 
 .. code-block:: yaml
 
@@ -71,7 +71,7 @@ deployment.
 
 To test it, you will need two virtual machines. Cirros does not support the Ceph
 kernel client, so you will need to use a different image. Any regular Linux
-distribution should work. As an example, we will use Ubuntu 20.04.
+distribution should work. As an example, this guide will use Ubuntu 20.04.
 
 Download the image locally:
 
@@ -133,7 +133,7 @@ Back on the host, install the Manila client:
 
 .. code-block:: bash
 
-      sudo dnf install -y python-manilaclient
+      pip install python-manilaclient
 
 Then create a share type and share:
 

From 2651d82f8a1bf1029ffde8b48e080cc70dd7f558 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 3 Apr 2023 12:12:46 +0100
Subject: [PATCH 07/14] CI: Add a push flag to kolla container image build
 workflow

---
 .../stackhpc-container-image-build.yml        | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 4f56eed5b..c5d7b425b 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -26,6 +26,11 @@ on:
         options:
           - centos
           - ubuntu
+      push:
+        description: Whether to push images
+        type: boolean
+        required: false
+        default: true
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -99,9 +104,13 @@ jobs:
 
       - name: Build and push kolla overcloud images
         run: |
+          args="${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }}"
+          if ${{ inputs.push }} == 'true'; then
+            args="$args --push"
+          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
+          kayobe overcloud container image build "$args"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
@@ -126,9 +135,13 @@ jobs:
 
       - name: Build and push kolla seed images
         run: |
+          args="kolla_base_distro=${{ inputs.distro }}"
+          if ${{ inputs.push }} == 'true'; then
+            args="$args --push"
+          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed container image build --push -e kolla_base_distro=${{ inputs.distro }}
+          kayobe seed container image build "$args"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'
@@ -159,7 +172,7 @@ jobs:
     name: Trigger container image repository sync
     needs:
       - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
     runs-on: ubuntu-latest
     steps:
       # NOTE(mgoddard): Trigger another CI workflow in the

From 09d701030c9045a40cc90e95c09a40eb089f9b50 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 3 Apr 2023 12:13:16 +0100
Subject: [PATCH 08/14] CI: Generate Kolla tag in advance

This ensures we use the same tag for seed & overcloud images.
---
 .../stackhpc-container-image-build.yml        | 41 ++++++++++++++++---
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index c5d7b425b..d466e635b 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -35,10 +35,41 @@ on:
 env:
   ANSIBLE_FORCE_COLOR: True
 jobs:
+  generate-tag:
+    name: Generate container image tag
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    permissions: {}
+    outputs:
+      openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
+      kolla_tag: ${{ steps.kolla_tag.outputs.kolla_tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Determine OpenStack release
+        id: openstack_release
+        run: |
+          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
+          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
+
+      # Generate a tag to apply to all built container images.
+      # Without this, each kayobe * container image build command would use a different tag.
+      - name: Generate container image tag
+        id: kolla_tag
+        run: |
+          echo "kolla_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
+      - name: Display container image tag
+        run: |
+          echo "${{ steps.kolla_tag.outputs.kolla_tag }}"
+
   container-image-build:
     name: Build Kolla container images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    needs:
+      - generate-tag
     steps:
       - uses: actions/checkout@v3
         with:
@@ -48,7 +79,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/wallaby
+          ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
           path: src/kayobe
 
       # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
@@ -104,13 +135,13 @@ jobs:
 
       - name: Build and push kolla overcloud images
         run: |
-          args="${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }}"
+          args="${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }} -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build "$args"
+          kayobe overcloud container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
@@ -135,13 +166,13 @@ jobs:
 
       - name: Build and push kolla seed images
         run: |
-          args="kolla_base_distro=${{ inputs.distro }}"
+          args="kolla_base_distro=${{ inputs.distro }} -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed container image build "$args"
+          kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'

From efa70f6741f89b5217b3df685ea85933b6f55119 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 3 Apr 2023 12:48:05 +0100
Subject: [PATCH 09/14] CI: Drop permissions for container image build job

---
 .github/workflows/stackhpc-container-image-build.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index d466e635b..a235893b2 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -68,6 +68,7 @@ jobs:
     name: Build Kolla container images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    permissions: {}
     needs:
       - generate-tag
     steps:
@@ -205,6 +206,7 @@ jobs:
       - container-image-build
     if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
     runs-on: ubuntu-latest
+    permissions: {}
     steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.

From 8c153a3b02fae4afae280b3de471ae33b64266ef Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 3 Apr 2023 13:51:24 +0100
Subject: [PATCH 10/14] CI: Support building multiple container image distros
 in parallel

---
 .../stackhpc-container-image-build.yml        | 81 ++++++++++---------
 1 file changed, 41 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index a235893b2..adef6ffdc 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -18,14 +18,16 @@ on:
         type: boolean
         required: false
         default: false
-      distro:
-        description: Container image OS distribution
-        type: choice
+      centos-stream-8:
+        description: Build CentOS Stream 8 images?
+        type: boolean
+        required: false
+        default: true
+      ubuntu-focal:
+        description: Build Ubuntu Focal 20.04 images?
+        type: boolean
         required: false
-        default: centos
-        options:
-          - centos
-          - ubuntu
+        default: true
       push:
         description: Whether to push images
         type: boolean
@@ -41,8 +43,9 @@ jobs:
     runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
     permissions: {}
     outputs:
-      openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
       kolla_tag: ${{ steps.kolla_tag.outputs.kolla_tag }}
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -60,6 +63,24 @@ jobs:
         run: |
           echo "kolla_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
 
+      # Dynamically define job matrix.
+      # We need a separate matrix entry for each distribution, when the relevant input is true.
+      # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
+      - name: Generate build matrix
+        id: set-matrix
+        run: |
+          comma=""
+          echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          if [[ ${{ inputs.centos-stream-8 }} == 'true' ]]; then
+            echo -n "$comma\"centos\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          if [[ ${{ inputs.ubuntu-focal }} == 'true' ]]; then
+            echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
+          echo "]}" >> $GITHUB_OUTPUT
+
       - name: Display container image tag
         run: |
           echo "${{ steps.kolla_tag.outputs.kolla_tag }}"
@@ -69,6 +90,8 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
     permissions: {}
+    strategy:
+      matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
     needs:
       - generate-tag
     steps:
@@ -136,7 +159,9 @@ jobs:
 
       - name: Build and push kolla overcloud images
         run: |
-          args="${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }} -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="${{ github.event.inputs.regexes }}"
+          args="$args -e kolla_base_distro=${{ matrix.distro }}"
+          args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -147,27 +172,10 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
 
-      - name: Display the overcloud container image tag
-        run: |
-          echo "$(cat ~/kolla_tag)"
-        if: github.event.inputs.overcloud == 'true'
-
-      - name: Get built overcloud container images
-        run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > overcloud-container-images
-        if: github.event.inputs.overcloud == 'true'
-
-      - name: Upload overcloud-container-images artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: Overcloud container images
-          path: overcloud-container-images
-          retention-days: 7
-        if: github.event.inputs.overcloud == 'true'
-
       - name: Build and push kolla seed images
         run: |
-          args="kolla_base_distro=${{ inputs.distro }} -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="kolla_base_distro=${{ matrix.distro }}"
+          args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -178,23 +186,16 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'
 
-      - name: Display the seed container image tag
-        run: |
-          echo "$(cat ~/kolla_tag)"
-        if: github.event.inputs.seed == 'true'
-
-      - name: Get built seed container images
+      - name: Get built container images
         run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > seed-container-images
-        if: github.event.inputs.seed == 'true'
+          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
 
-      - name: Upload seed-container-images artifact
+      - name: Upload container images artifact
         uses: actions/upload-artifact@v3
         with:
-          name: Seed container images
-          path: seed-container-images
+          name: ${{ matrix.distro }} container images
+          path: ${{ matrix.distro }}-container-images
           retention-days: 7
-        if: github.event.inputs.seed == 'true'
 
       - name: Prune local Kolla container images over 1 week old
         run: |

From a6157ccf7bde7702277e1654320a4a2165d38f23 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 3 Apr 2023 14:47:53 +0100
Subject: [PATCH 11/14] CI: Drop permissions for package promote job

---
 .github/workflows/stackhpc-promote.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml
index 81b05b774..978cab437 100644
--- a/.github/workflows/stackhpc-promote.yml
+++ b/.github/workflows/stackhpc-promote.yml
@@ -11,6 +11,7 @@ jobs:
     name: Trigger package repository promotion
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-latest
+    permissions: {}
     steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.

From 6176037d74a32c21c714044bbc015603494e0cd6 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Tue, 4 Apr 2023 13:03:52 +0200
Subject: [PATCH 12/14] docs: fix typo

---
 doc/source/configuration/lvm.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/configuration/lvm.rst b/doc/source/configuration/lvm.rst
index 65b892aa8..a96ca8db9 100644
--- a/doc/source/configuration/lvm.rst
+++ b/doc/source/configuration/lvm.rst
@@ -76,7 +76,7 @@ another LV needs to be grown at a later date.
 Growroot playbook
 =================
 
-A ``growroot.yml`` custom playbook is provied that can be used to grow the
+A ``growroot.yml`` custom playbook is provided that can be used to grow the
 partition and LVM Physical Volume (PV) of the root Volume Group (VG). This
 allows for expansion of Logical Volumes (LVs) in that VG.
 

From 977d58b5f717344ae97103893c7b0984f9d3c623 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 4 Apr 2023 16:04:38 +0100
Subject: [PATCH 13/14] Remove write-kolla-tag.yml custom playbook & hooks

It is no longer necessary since we generate a tag in advance.
---
 README.rst                                    |  5 +----
 etc/kayobe/ansible/write-kolla-tag.yml        | 19 -------------------
 .../post.d/50-write-kolla-tag.yml             |  1 -
 .../post.d/50-write-kolla-tag.yml             |  1 -
 4 files changed, 1 insertion(+), 25 deletions(-)
 delete mode 100644 etc/kayobe/ansible/write-kolla-tag.yml
 delete mode 120000 etc/kayobe/hooks/overcloud-container-image-build/post.d/50-write-kolla-tag.yml
 delete mode 120000 etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml

diff --git a/README.rst b/README.rst
index d8e39fa87..76255e096 100644
--- a/README.rst
+++ b/README.rst
@@ -459,10 +459,7 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
-The container images are tagged as ``wallaby-<datetime>``. This Kayobe
-configuration includes a hook that writes the tag to ``~/kolla_tag``, since
-it is not always simple to determine which tag was last applied to built
-images.
+The container images are tagged as ``wallaby-<datetime>``.
 
 To use the new images, edit
 ``~/src/kayobe-config/etc/kayobe/kolla.yml`` to set the above
diff --git a/etc/kayobe/ansible/write-kolla-tag.yml b/etc/kayobe/ansible/write-kolla-tag.yml
deleted file mode 100644
index e0099bb87..000000000
--- a/etc/kayobe/ansible/write-kolla-tag.yml
+++ /dev/null
@@ -1,19 +0,0 @@
----
-- name: Write out value of kolla_tag
-  hosts: localhost
-  gather_facts: no
-  vars:
-    kolla_tag_dest: ~/kolla_tag
-  tasks:
-    - name: Query image tag
-      shell:
-        cmd: >-
-          grep '^tag' {{ kolla_build_config_path }}/kolla-build.conf |
-          sed 's/tag\s=\s\(.*\)/\1/'
-      become: true
-      register: tag
-
-    - name: Write out value of kolla_tag
-      copy:
-        content: "{{ tag.stdout }}"
-        dest: "{{ kolla_tag_dest }}"
diff --git a/etc/kayobe/hooks/overcloud-container-image-build/post.d/50-write-kolla-tag.yml b/etc/kayobe/hooks/overcloud-container-image-build/post.d/50-write-kolla-tag.yml
deleted file mode 120000
index c685974a2..000000000
--- a/etc/kayobe/hooks/overcloud-container-image-build/post.d/50-write-kolla-tag.yml
+++ /dev/null
@@ -1 +0,0 @@
-../../../ansible/write-kolla-tag.yml
\ No newline at end of file
diff --git a/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml b/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml
deleted file mode 120000
index c685974a2..000000000
--- a/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml
+++ /dev/null
@@ -1 +0,0 @@
-../../../ansible/write-kolla-tag.yml
\ No newline at end of file

From d4bb26f513deeaa164fc6a953c5bc45c12f02615 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 12 Apr 2023 15:26:30 +0100
Subject: [PATCH 14/14] CI: Add Rocky Linux 9 to Kolla container build workflow

---
 .github/workflows/stackhpc-container-image-build.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 01485c256..ea38864d4 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -23,6 +23,11 @@ on:
         type: boolean
         required: false
         default: true
+      rocky-linux-9:
+        description: Build Rocky Linux 9 images?
+        type: boolean
+        required: false
+        default: true
       ubuntu-focal:
         description: Build Ubuntu Focal 20.04 images?
         type: boolean
@@ -75,6 +80,10 @@ jobs:
             echo -n "$comma\"centos\"" >> $GITHUB_OUTPUT
             comma=", "
           fi
+          if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
+            echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
+            comma=", "
+          fi
           if [[ ${{ inputs.ubuntu-focal }} == 'true' ]]; then
             echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
             comma=", "