From 4aa8aed69c4bcf0ef469156a15eb2dbb6d106c47 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Mon, 22 Aug 2022 14:44:27 +0000
Subject: [PATCH 01/21] feat: create `ci-multinode` environment

The `ci-multinode` environment is a clone of the `ci-aio` environment
and as a result will need adapting.
---
 .../environments/ci-multinode/globals.yml     |  63 ++++++++++
 .../group_vars/controllers/network-interfaces |  12 ++
 .../ci-multinode/inventory/groups             |   5 +
 .../environments/ci-multinode/inventory/hosts |   6 +
 .../ci-multinode/inventory/kayobe-automation  |   2 +
 .../ci-multinode/kolla/config/ironic.conf     |   2 +
 .../ci-multinode/kolla/globals.yml            |  10 ++
 .../ci-multinode/network-allocation.yml       |   0
 .../environments/ci-multinode/networks.yml    | 119 ++++++++++++++++++
 .../environments/ci-multinode/stackhpc-ci.yml |  58 +++++++++
 10 files changed, 277 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/globals.yml
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/groups
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/hosts
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
 create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf
 create mode 100644 etc/kayobe/environments/ci-multinode/kolla/globals.yml
 create mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml
 create mode 100644 etc/kayobe/environments/ci-multinode/networks.yml
 create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml

diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml
new file mode 100644
index 000000000..3a6674de3
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/globals.yml
@@ -0,0 +1,63 @@
+---
+# Kayobe global configuration.
+
+###############################################################################
+# Local path configuration (Ansible control host).
+
+# Path to Kayobe configuration directory on Ansible control host, with an
+# environment path appended if kayobe_environment is set.
+#kayobe_config_path:
+
+# Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an
+# empty string if $KAYOBE_ENVIRONMENT is not set. Can also be set via the
+# --environment argument when invoking kayobe.
+#kayobe_environment:
+
+# Path to Kayobe configuration directory on Ansible control host with an
+# environment path appended if kayobe_environment is set.
+#kayobe_env_config_path:
+
+###############################################################################
+# Remote path configuration (seed, seed-hypervisor and overcloud hosts).
+
+# Base path for kayobe state on remote hosts.
+#base_path:
+
+# Path in which to store configuration on remote hosts.
+#config_path:
+
+# Path in which to cache downloaded images on remote hosts.
+#image_cache_path:
+
+# Path on which to checkout source code repositories on remote hosts.
+#source_checkout_path:
+
+# Path on which to create python virtualenvs on remote hosts.
+#virtualenv_path:
+
+###############################################################################
+# User configuration.
+
+# User with which to access remote hosts. This user will be created if it does
+# not exist.
+#kayobe_ansible_user:
+
+###############################################################################
+# OS distribution.
+
+# OS distribution name. Valid options are "centos", "ubuntu". Default is
+# "centos".
+os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}"
+
+# OS release. Valid options are "8-stream" when os_distribution is "centos", or
+# "focal" when os_distribution is "ubuntu".
+#os_release:
+
+###############################################################################
+
+# Avoid a reboot.
+disable_selinux_do_reboot: false
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
new file mode 100644
index 000000000..cae8e6a7d
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
@@ -0,0 +1,12 @@
+---
+###############################################################################
+# Network interface definitions for the controller group.
+
+# Controller interface on all-in-one network.
+aio_interface: breth1
+# Use dummy1 if it exists, otherwise the bridge will have no ports.
+aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/inventory/groups b/etc/kayobe/environments/ci-multinode/inventory/groups
new file mode 100644
index 000000000..43967dac1
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/groups
@@ -0,0 +1,5 @@
+[controllers]
+
+[container-image-builders:children]
+# Build container images on the all-in-one controller.
+controllers
diff --git a/etc/kayobe/environments/ci-multinode/inventory/hosts b/etc/kayobe/environments/ci-multinode/inventory/hosts
new file mode 100644
index 000000000..b041573df
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/hosts
@@ -0,0 +1,6 @@
+# A single all-in-one controller/compute host.
+[controllers]
+controller0
+
+[compute:children]
+controllers
diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
new file mode 100644
index 000000000..5ab47462a
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
@@ -0,0 +1,2 @@
+[tempest_runner]
+controller0
diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf
new file mode 100644
index 000000000..45f848825
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf
@@ -0,0 +1,2 @@
+[DEFAULT]
+minimum_required_memory = 256
diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
new file mode 100644
index 000000000..9c769d3ea
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
@@ -0,0 +1,10 @@
+---
+# Most development environments will use nested virtualisation, and we can't
+# guarantee that nested KVM support is available. Use QEMU as a lowest common
+# denominator.
+nova_compute_virt_type: qemu
+
+# Reduce the control plane's memory footprint by limiting the number of worker
+# processes to one per-service.
+openstack_service_workers: "1"
+openstack_service_rpc_workers: "1"
diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml
new file mode 100644
index 000000000..a42719638
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/networks.yml
@@ -0,0 +1,119 @@
+---
+# Kayobe network configuration.
+
+###############################################################################
+# Network role to network mappings.
+
+# Map all networks to the all-in-one network.
+
+# Name of the network used for admin access to the overcloud
+#admin_oc_net_name:
+admin_oc_net_name: admin
+
+# Name of the network used by the seed to manage the bare metal overcloud
+# hosts via their out-of-band management controllers.
+#oob_oc_net_name:
+oob_oc_net_name: aio
+
+# Name of the network used by the seed to provision the bare metal overcloud
+# hosts.
+#provision_oc_net_name:
+provision_oc_net_name: aio
+
+# Name of the network used by the overcloud hosts to manage the bare metal
+# compute hosts via their out-of-band management controllers.
+#oob_wl_net_name:
+oob_wl_net_name: aio
+
+# Name of the network used by the overcloud hosts to provision the bare metal
+# workload hosts.
+#provision_wl_net_name:
+provision_wl_net_name: aio
+
+# Name of the network used to expose the internal OpenStack API endpoints.
+#internal_net_name:
+internal_net_name: aio
+
+# List of names of networks used to provide external network access via
+# Neutron.
+# Deprecated name: external_net_name
+# If external_net_name is defined, external_net_names will default to a list
+# containing one item, external_net_name.
+#external_net_names:
+external_net_names:
+  - aio
+
+# Name of the network used to expose the public OpenStack API endpoints.
+#public_net_name:
+public_net_name: aio
+
+# Name of the network used by Neutron to carry tenant overlay network traffic.
+#tunnel_net_name:
+tunnel_net_name: aio
+
+# Name of the network used to carry storage data traffic.
+#storage_net_name:
+storage_net_name: aio
+
+# Name of the network used to carry storage management traffic.
+#storage_mgmt_net_name:
+storage_mgmt_net_name: aio
+
+# Name of the network used to carry swift storage data traffic.
+#swift_storage_net_name:
+swift_storage_net_name: aio
+
+# Name of the network used to carry swift storage replication traffic.
+#swift_storage_replication_net_name:
+swift_storage_replication_net_name: aio
+
+# Name of the network used to perform hardware introspection on the bare metal
+# workload hosts.
+#inspection_net_name:
+inspection_net_name: aio
+
+# Name of the network used to perform cleaning on the bare metal workload
+# hosts
+#cleaning_net_name:
+cleaning_net_name: aio
+
+###############################################################################
+# Network definitions.
+
+# All-in-one network.
+aio_cidr: 192.168.33.0/24
+aio_allocation_pool_start: 192.168.33.3
+aio_allocation_pool_end: 192.168.33.30
+aio_neutron_allocation_pool_start: 192.168.33.31
+aio_neutron_allocation_pool_end: 192.168.33.127
+aio_inspection_allocation_pool_start: 192.168.33.128
+aio_inspection_allocation_pool_end: 192.168.33.254
+aio_vip_address: 192.168.33.2
+
+###############################################################################
+# Network virtual patch link configuration.
+
+# Suffix for Open vSwitch bridge names.
+#network_bridge_suffix_ovs:
+
+# Prefix for virtual patch interface names.
+#network_patch_prefix:
+
+# Suffix for virtual patch link interface names when connected towards the
+# physical interface.
+#network_patch_suffix_phy:
+
+# Suffix for virtual patch link interface names when connected towards the
+# OVS bridge.
+#network_patch_suffix_ovs:
+
+###############################################################################
+# Network routing table configuration.
+
+# List of IP routing tables. Each item should be a dict containing 'id' and
+# 'name' items. These tables will be added to /etc/iproute2/rt_tables.
+#network_route_tables:
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
new file mode 100644
index 000000000..b78339cdb
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -0,0 +1,58 @@
+---
+###############################################################################
+# Kolla configuration.
+
+# Docker namespace to use for Kolla images. Default is 'kolla'.
+kolla_docker_namespace: stackhpc-dev
+
+###############################################################################
+# Network configuration.
+
+# Don't touch resolv.conf: use Neutron DNS for accessing Pulp server via
+# hostname.
+resolv_is_managed: false
+
+###############################################################################
+# StackHPC configuration.
+
+# Host and port of a package repository mirror.
+# Build and deploy the development Pulp service repositories.
+stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080"
+
+# Build and deploy released Pulp repository versions.
+stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
+stackhpc_repo_centos_stream_appstream_version: "{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}"
+stackhpc_repo_centos_stream_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}"
+stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}"
+stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}"
+stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}"
+stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}"
+stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}"
+stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}"
+stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}"
+stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}"
+stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}"
+stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
+stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
+stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
+stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
+stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
+stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
+
+
+# Host and port of container registry.
+# Push built images to the development Pulp service registry.
+stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
+
+# Username and password of container registry.
+stackhpc_docker_registry_username: "admin"
+stackhpc_docker_registry_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  38653432663133653430323937626239323836653537633633326265643930653034306532646464
+  3766353637326534643136326533633034666535323364660a633639393433376564363666663233
+  38333133393730633666613965653364316162353337313330346164303631313731646461363461
+  3963323635373866630a633533376339363734626664333765313665623662613764363038383735
+  38646138376438643533376161376634653439386230353365316239613430363338

From 6f49c3f09f2c04ab8a3fbe94804ad4e0a6bef389 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Wed, 24 Aug 2022 16:57:54 +0000
Subject: [PATCH 02/21] feat: successfully configure two controllers

---
 .../group_vars/compute/network-interfaces          | 14 ++++++++++++++
 .../group_vars/controllers/network-interfaces      |  2 ++
 .../inventory/group_vars/seed/network-interfaces   | 14 ++++++++++++++
 .../environments/ci-multinode/inventory/hosts      |  6 ------
 .../ci-multinode/network-allocation.yml            |  0
 .../environments/ci-multinode/stackhpc-ci.yml      | 10 ++--------
 6 files changed, 32 insertions(+), 14 deletions(-)
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
 delete mode 100644 etc/kayobe/environments/ci-multinode/inventory/hosts
 delete mode 100644 etc/kayobe/environments/ci-multinode/network-allocation.yml

diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
new file mode 100644
index 000000000..016f1e72b
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
@@ -0,0 +1,14 @@
+---
+###############################################################################
+# Network interface definitions for the controller group.
+
+admin_interface: eno1
+
+# Controller interface on all-in-one network.
+aio_interface: breth1
+# Use dummy1 if it exists, otherwise the bridge will have no ports.
+aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
index cae8e6a7d..016f1e72b 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
@@ -2,6 +2,8 @@
 ###############################################################################
 # Network interface definitions for the controller group.
 
+admin_interface: eno1
+
 # Controller interface on all-in-one network.
 aio_interface: breth1
 # Use dummy1 if it exists, otherwise the bridge will have no ports.
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
new file mode 100644
index 000000000..092b82f09
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
@@ -0,0 +1,14 @@
+---
+###############################################################################
+# Network interface definitions for the controller group.
+
+admin_interface: eth0
+
+# Controller interface on all-in-one network.
+aio_interface: breth1
+# Use dummy1 if it exists, otherwise the bridge will have no ports.
+aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/inventory/hosts b/etc/kayobe/environments/ci-multinode/inventory/hosts
deleted file mode 100644
index b041573df..000000000
--- a/etc/kayobe/environments/ci-multinode/inventory/hosts
+++ /dev/null
@@ -1,6 +0,0 @@
-# A single all-in-one controller/compute host.
-[controllers]
-controller0
-
-[compute:children]
-controllers
diff --git a/etc/kayobe/environments/ci-multinode/network-allocation.yml b/etc/kayobe/environments/ci-multinode/network-allocation.yml
deleted file mode 100644
index e69de29bb..000000000
diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index b78339cdb..6f2eb4599 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -48,11 +48,5 @@ stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
 
 # Username and password of container registry.
-stackhpc_docker_registry_username: "admin"
-stackhpc_docker_registry_password: !vault |
-  $ANSIBLE_VAULT;1.1;AES256
-  38653432663133653430323937626239323836653537633633326265643930653034306532646464
-  3766353637326534643136326533633034666535323364660a633639393433376564363666663233
-  38333133393730633666613965653364316162353337313330346164303631313731646461363461
-  3963323635373866630a633533376339363734626664333765313665623662613764363038383735
-  38646138376438643533376161376634653439386230353365316239613430363338
+stackhpc_docker_registry_username: ""
+stackhpc_docker_registry_password: ""

From d057d07b34e1fe343da2be8b1bcbc82fbea127f5 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Thu, 25 Aug 2022 10:58:17 +0000
Subject: [PATCH 03/21] feat: succesfully configure three controllers with
 VXLAN

---
 etc/kayobe/ansible/configure-vxlan.yml        | 20 +++++++++++++++++++
 etc/kayobe/ansible/requirements.yml           |  4 ++++
 .../group_vars/controllers/network-interfaces |  2 +-
 .../post.d/50-configure-vxlan.yml             |  1 +
 4 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 etc/kayobe/ansible/configure-vxlan.yml
 create mode 120000 etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
new file mode 100644
index 000000000..fed1b05a5
--- /dev/null
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -0,0 +1,20 @@
+---
+- name: Configure VXLAN
+  hosts: controllers
+  gather_facts: false
+  vars:
+    ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
+    # Work around no known_hosts entry on first boot.
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+  roles:
+    - role: vxlan
+      vars:
+        vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}"
+        vxlan_interfaces:
+          - device: vxlan10
+            group: 224.0.0.10
+            bridge: breth1
+
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 99a0c83f7..d3e99dfb0 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -4,3 +4,7 @@ collections:
     version: 1.10.0
   - name: stackhpc.pulp
     version: 0.3.0
+roles:
+  - name: vxlan
+    src: https://github.com/stackhpc/ansible-role-vxlan.git
+    version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
index 016f1e72b..2e94b6c2d 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces
@@ -7,7 +7,7 @@ admin_interface: eno1
 # Controller interface on all-in-one network.
 aio_interface: breth1
 # Use dummy1 if it exists, otherwise the bridge will have no ports.
-aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
+aio_bridge_ports: []
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml
new file mode 120000
index 000000000..d8d2ce4bb
--- /dev/null
+++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml
@@ -0,0 +1 @@
+../../../ansible/configure-vxlan.yml
\ No newline at end of file

From 27e6c619aa4db153159280565e2b7ec3d0b067a5 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Wed, 31 Aug 2022 12:01:07 +0000
Subject: [PATCH 04/21] feat: add `kayobe-compute-01`

---
 etc/kayobe/ansible/configure-vxlan.yml                          | 2 +-
 .../inventory/group_vars/compute/network-interfaces             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
index fed1b05a5..a226927f1 100644
--- a/etc/kayobe/ansible/configure-vxlan.yml
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -1,6 +1,6 @@
 ---
 - name: Configure VXLAN
-  hosts: controllers
+  hosts: controllers,compute
   gather_facts: false
   vars:
     ansible_user: "{{ bootstrap_user }}"
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
index 016f1e72b..2e94b6c2d 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces
@@ -7,7 +7,7 @@ admin_interface: eno1
 # Controller interface on all-in-one network.
 aio_interface: breth1
 # Use dummy1 if it exists, otherwise the bridge will have no ports.
-aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
+aio_bridge_ports: []
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

From b99dfa65c88c8458c5a9dd7ff58797883907db29 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Wed, 31 Aug 2022 12:02:16 +0000
Subject: [PATCH 05/21] feat: add `growroot` playbook for successful deployment

---
 etc/kayobe/environments/ci-multinode/compute.yml     | 2 ++
 etc/kayobe/environments/ci-multinode/controllers.yml | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/compute.yml
 create mode 100644 etc/kayobe/environments/ci-multinode/controllers.yml

diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml
new file mode 100644
index 000000000..254afc8e4
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/compute.yml
@@ -0,0 +1,2 @@
+compute_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"
diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml
new file mode 100644
index 000000000..9198a4d4c
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/controllers.yml
@@ -0,0 +1,2 @@
+controller_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

From 5b55f69713b099708b994595ddf06a6917e85dd0 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Fri, 2 Sep 2022 15:05:04 +0000
Subject: [PATCH 06/21] feat: successfully deploy environment

This configuration is capable of successfully deploying a multinode
cluster across three controllers and one compute node.
---
 etc/kayobe/ansible/configure-vxlan.yml                 |  3 ++-
 .../inventory/group_vars/seed/network-interfaces       |  3 +--
 .../ci-multinode/inventory/kayobe-automation           |  2 +-
 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml   | 10 ++++++++--
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
index a226927f1..92b699615 100644
--- a/etc/kayobe/ansible/configure-vxlan.yml
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -1,6 +1,6 @@
 ---
 - name: Configure VXLAN
-  hosts: controllers,compute
+  hosts: seed,controllers,compute
   gather_facts: false
   vars:
     ansible_user: "{{ bootstrap_user }}"
@@ -13,6 +13,7 @@
     - role: vxlan
       vars:
         vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}"
+        vxlan_dstport: 4790
         vxlan_interfaces:
           - device: vxlan10
             group: 224.0.0.10
diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
index 092b82f09..13f7aa8ea 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces
@@ -7,8 +7,7 @@ admin_interface: eth0
 # Controller interface on all-in-one network.
 aio_interface: breth1
 # Use dummy1 if it exists, otherwise the bridge will have no ports.
-aio_bridge_ports: "{{ ['dummy1'] if 'ansible_dummy1' in hostvars[inventory_hostname] else [] }}"
-
+aio_bridge_ports: []
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
index 5ab47462a..5ac95a0d2 100644
--- a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
+++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation
@@ -1,2 +1,2 @@
 [tempest_runner]
-controller0
+kayobe-controller-01
diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index 6f2eb4599..191b9a806 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -48,5 +48,11 @@ stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
 
 # Username and password of container registry.
-stackhpc_docker_registry_username: ""
-stackhpc_docker_registry_password: ""
+stackhpc_docker_registry_username: "stackhpc-kayobe-ci"
+stackhpc_docker_registry_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          30333932623139383562343036366566363230656536376164633463366235623463323937303135
+          6533333638613764383136613730346436643831646530640a363262373964323461616638636135
+          65623338333236323030346564363035313462333335333032643661363834356432363833303838
+          3936616462303062350a316165396265313766316338303338336132316264316631376163643133
+          33616239633435636630373039616630323163343962336330303864626561303238

From c35932403b4ced0f558ae78245659038b05674a4 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Mon, 12 Sep 2022 15:22:43 +0100
Subject: [PATCH 07/21] fix: syntax issues raised by `yamllint`

---
 etc/kayobe/ansible/configure-vxlan.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
index 92b699615..12deba374 100644
--- a/etc/kayobe/ansible/configure-vxlan.yml
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -18,4 +18,3 @@
           - device: vxlan10
             group: 224.0.0.10
             bridge: breth1
-

From 8eb2f85f64c67be84951741dab19309412475357 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 3 Oct 2022 15:56:18 +0100
Subject: [PATCH 08/21] added notes

---
 .../environments/ci-multinode/README.md       | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/README.md

diff --git a/etc/kayobe/environments/ci-multinode/README.md b/etc/kayobe/environments/ci-multinode/README.md
new file mode 100644
index 000000000..668e8895d
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/README.md
@@ -0,0 +1,52 @@
+# Multinode Test Environment 
+
+## Set up hosts
+1. Create four baremetal instances with a centos 8 stream LVM image, and a 
+Centos 8 stream vm
+2. SSH into each baremetal and run `sudo chown -R centos:.` in the home directory,
+   then add the lines 
+   >`10.0.0.34 pelican pelican.service.compute.sms-lab.cloud`  
+   >`10.205.3.187 pulp-server pulp-server.internal.sms-cloud`  
+
+   to /etc/hosts (if you're waiting on them starting up, you can progress until 
+   `kayobe overcloud host configure` without this step)
+
+## Basic Kayobe Setup
+1. SSH into the VM
+1. `sudo dnf install -y python3-virtualenv`
+1. `mkdir src` and `cd src`
+1. Clone https://github.com/stackhpc/stackhpc-kayobe-config.git, then checkout
+   commit f31df6256f1b1fea99c84547d44f06c4cb74b161 
+1. `cd ..` and `mkdir venvs`
+1. `virtualenv venvs/kayobe` and source `venvs/kayobe/bin/activate`
+1. `pip install -U pip`
+1. `pip install ./src/kayobe`
+1. Acquire the Ansible Vault password for this repository, and store a copy at
+``~/vault-pw``
+1. `export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw)`
+
+## Config changes
+1. In etc/kayobe/ansible/requirements.yml remove version from vxlan
+4. In etc/kayobe/ansible/configure-vxlan.yml, change the group of
+vxlan_interfaces so that the last octet is different e.g. 224.0.0.15
+5. Also under vxlan_interfaces, add vni:x where x is between 500 and 1000
+5. Also under vxlan_interfaces, check vxlan_dstport is not 4789 (this causes
+conflicts, change to 4790)
+6. In /etc/kayobe/environments/ci-multinode/tf-networks.yml, 
+   edit admin_ips so that the compute and controller IPs line up with the 
+   instances that were created earlier, remove the other IPs for seed and 
+   cephOSD
+7. In /etc/kayobe/environments/ci-multinode/network-allocation.yml, remove all
+the entries and just assign `aio_ips:` an empty set `[]`
+8. In etc/kayobe/environments/ci-multinode/inventory/hosts, remove the seed
+9. run stackhpc-kayobe-config/etc/kayobe/ansible/growroot.yml (if this fails,
+manually increase the partition size on each host)
+
+## Final steps
+1. `source kayobe-env --environment ci-aio`
+10. Run `kayobe overcloud host configure`
+11. Run `kayobe overcloud service deploy`
+
+TODO:
+- openstack will use the port that as been reserved for vslan so 4790 was chosen because it's one away
+-m 

From ed3f2869d7ed9d3b53d4182a08a6ed48caf82a47 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 3 Oct 2022 16:00:36 +0100
Subject: [PATCH 09/21] removed TODO

---
 etc/kayobe/environments/ci-multinode/README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/etc/kayobe/environments/ci-multinode/README.md b/etc/kayobe/environments/ci-multinode/README.md
index 668e8895d..c2511150c 100644
--- a/etc/kayobe/environments/ci-multinode/README.md
+++ b/etc/kayobe/environments/ci-multinode/README.md
@@ -46,7 +46,3 @@ manually increase the partition size on each host)
 1. `source kayobe-env --environment ci-aio`
 10. Run `kayobe overcloud host configure`
 11. Run `kayobe overcloud service deploy`
-
-TODO:
-- openstack will use the port that as been reserved for vslan so 4790 was chosen because it's one away
--m 

From 08df28022e2dea99c4d1f2b75257b12796731686 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Wed, 28 Sep 2022 15:01:28 +0000
Subject: [PATCH 10/21] feat: update vxlan role version and expand hosts

---
 etc/kayobe/ansible/configure-vxlan.yml | 2 +-
 etc/kayobe/ansible/requirements.yml    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
index 12deba374..924fdd7ad 100644
--- a/etc/kayobe/ansible/configure-vxlan.yml
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -1,6 +1,6 @@
 ---
 - name: Configure VXLAN
-  hosts: seed,controllers,compute
+  hosts: storage,seed,controllers,compute
   gather_facts: false
   vars:
     ansible_user: "{{ bootstrap_user }}"
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index d3e99dfb0..4edd6b905 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -7,4 +7,3 @@ collections:
 roles:
   - name: vxlan
     src: https://github.com/stackhpc/ansible-role-vxlan.git
-    version: b506e9817e1eb0a81ea66e0cddc40ab2ff66bc79

From a4eccccd93c49c935a6b71186ac99084e8552713 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Sun, 2 Oct 2022 14:33:40 +0000
Subject: [PATCH 11/21] feat: enable ceph and ceph related services

---
 etc/kayobe/environments/ci-multinode/kolla.yml         | 3 +++
 etc/kayobe/environments/ci-multinode/kolla/globals.yml | 9 +++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/kolla.yml

diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml
new file mode 100644
index 000000000..168500c8b
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/kolla.yml
@@ -0,0 +1,3 @@
+---
+kolla_enable_cinder: true
+kolla_enable_cinder_backup: true
diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
index 9c769d3ea..ebfe64897 100644
--- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml
+++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
@@ -8,3 +8,12 @@ nova_compute_virt_type: qemu
 # processes to one per-service.
 openstack_service_workers: "1"
 openstack_service_rpc_workers: "1"
+
+# Glance Ceph configuration
+glance_backend_ceph: "yes"
+
+# Cinder Ceph configuration
+cinder_backend_ceph: "yes"
+
+# Nova Ceph configuration
+nova_backend_ceph: "yes"

From 483b887a5b53948341823d81aecf12c69c3544a7 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Mon, 3 Oct 2022 15:30:50 +0100
Subject: [PATCH 12/21] feat: pull vxlan from ansible galaxy

---
 etc/kayobe/ansible/configure-vxlan.yml | 2 +-
 etc/kayobe/ansible/requirements.yml    | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml
index 924fdd7ad..f07933c1b 100644
--- a/etc/kayobe/ansible/configure-vxlan.yml
+++ b/etc/kayobe/ansible/configure-vxlan.yml
@@ -10,7 +10,7 @@
     # Work around no known_hosts entry on first boot.
     ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
   roles:
-    - role: vxlan
+    - role: stackhpc.vxlan
       vars:
         vxlan_phys_dev: "{{ admin_oc_net_name | net_interface }}"
         vxlan_dstport: 4790
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 4edd6b905..bd4d27e3f 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -5,5 +5,4 @@ collections:
   - name: stackhpc.pulp
     version: 0.3.0
 roles:
-  - name: vxlan
-    src: https://github.com/stackhpc/ansible-role-vxlan.git
+  - src: stackhpc.vxlan

From cbffc6e899816742fe9d7a5e458f70db5dd5684c Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzkoper@stackhpc.com>
Date: Wed, 5 Oct 2022 15:16:29 +0200
Subject: [PATCH 13/21] Removing vxlan hook

---
 .../hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml | 1 -
 1 file changed, 1 deletion(-)
 delete mode 120000 etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml

diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml
deleted file mode 120000
index d8d2ce4bb..000000000
--- a/etc/kayobe/hooks/overcloud-host-configure/post.d/50-configure-vxlan.yml
+++ /dev/null
@@ -1 +0,0 @@
-../../../ansible/configure-vxlan.yml
\ No newline at end of file

From 700b4dbb7cacdf20ea215885e28198e8567d4359 Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzkoper@stackhpc.com>
Date: Wed, 5 Oct 2022 15:21:59 +0200
Subject: [PATCH 14/21] added cephadm.yml to multinode environment.

---
 .../environments/ci-multinode/cephadm.yml     | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/cephadm.yml

diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml
new file mode 100644
index 000000000..812f89a06
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/cephadm.yml
@@ -0,0 +1,62 @@
+---
+###############################################################################
+# Cephadm deployment configuration.
+
+# Ceph container image.
+cephadm_image: "quay.io/ceph/ceph:v16.2.5"
+
+# Ceph OSD specification.
+cephadm_osd_spec:
+  service_type: osd
+  service_id: osd_spec_default
+  placement:
+    host_pattern: "*"
+  data_devices:
+    all: true
+
+###############################################################################
+# Ceph post-deployment configuration.
+
+# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role
+# for format.
+cephadm_ec_profiles: []
+
+# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format.
+cephadm_crush_rules: []
+
+# List of Ceph pools. See stackhpc.cephadm.pools role for format.
+cephadm_pools:
+  - name: backups
+    application: rbd
+    state: present
+  - name: images
+    application: rbd
+    state: present
+  - name: volumes
+    application: rbd
+    state: present
+  - name: vms
+    application: rbd
+    state: present
+
+# List of Cephx keys. See stackhpc.cephadm.keys role for format.
+cephadm_keys:
+  - name: client.cinder
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images"
+      mgr: "profile rbd pool=volumes, profile rbd pool=vms"
+    state: present
+  - name: client.cinder-backup
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=backups"
+      mgr: "profile rbd pool=volumes, profile rbd pool=backups"
+    state: present
+  - name: client.glance
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=images"
+      mgr: "profile rbd pool=images"
+    state: present
+

From 21ffab9b8f8de585af9ec6cbe52acbfd82d7ecda Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Wed, 5 Oct 2022 15:20:03 +0100
Subject: [PATCH 15/21] feat: add storage group vars

---
 .../group_vars/storage/network-interfaces          | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces

diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces
new file mode 100644
index 000000000..35000577f
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces
@@ -0,0 +1,14 @@
+---
+###############################################################################
+# Network interface definitions for the stroage group.
+
+admin_interface: ens3
+
+# Controller interface on all-in-one network.
+aio_interface: breth1
+# Use dummy1 if it exists, otherwise the bridge will have no ports.
+aio_bridge_ports: []
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

From 6c805de8c84627abf516fce0f3ab48998edee836 Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzk@stackhpc.com>
Date: Mon, 10 Oct 2022 12:55:41 +0200
Subject: [PATCH 16/21] Adding LVM configuration for storage nodes

---
 .../inventory/group_vars/storage/lvm          | 23 +++++++++++++++++++
 .../environments/ci-multinode/storage.yml     |  5 ++++
 2 files changed, 28 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm
 create mode 100644 etc/kayobe/environments/ci-multinode/storage.yml

diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm
new file mode 100644
index 000000000..03b356a9e
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm
@@ -0,0 +1,23 @@
+# StackHPC LVM lv_swap LV size.
+stackhpc_lvm_lv_swap_size: 1g
+
+# StackHPC LVM lv_root LV size.
+stackhpc_lvm_lv_root_size: 8g
+
+# StackHPC LVM lv_tmp LV size.
+stackhpc_lvm_lv_tmp_size: 2g
+
+# StackHPC LVM lv_var LV size.
+stackhpc_lvm_lv_var_size: 12g
+
+# StackHPC LVM lv_var_tmp LV size.
+stackhpc_lvm_lv_var_tmp_size: 1g
+
+# StackHPC LVM lv_log LV size.
+stackhpc_lvm_lv_log_size: 2g
+
+# StackHPC LVM lv_audit LV size.
+stackhpc_lvm_lv_audit_size: 1g
+
+# StackHPC LVM lv_home LV size.
+stackhpc_lvm_lv_home_size: 2g
diff --git a/etc/kayobe/environments/ci-multinode/storage.yml b/etc/kayobe/environments/ci-multinode/storage.yml
new file mode 100644
index 000000000..44b655ce7
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/storage.yml
@@ -0,0 +1,5 @@
+---
+# List of storage volume groups. See mrlesmithjr.manage-lvm role for
+# format.
+storage_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

From 54dd5be1f6d3b02f37b0c69cdf265e9f593da169 Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzk@stackhpc.com>
Date: Mon, 10 Oct 2022 17:49:20 +0200
Subject: [PATCH 17/21] Reencrypting docker_registry_password

---
 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index 191b9a806..60c5eef5c 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -51,8 +51,8 @@ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?:
 stackhpc_docker_registry_username: "stackhpc-kayobe-ci"
 stackhpc_docker_registry_password: !vault |
           $ANSIBLE_VAULT;1.1;AES256
-          30333932623139383562343036366566363230656536376164633463366235623463323937303135
-          6533333638613764383136613730346436643831646530640a363262373964323461616638636135
-          65623338333236323030346564363035313462333335333032643661363834356432363833303838
-          3936616462303062350a316165396265313766316338303338336132316264316631376163643133
-          33616239633435636630373039616630323163343962336330303864626561303238
+          33356166343730633865363431306535613736663764373034396132356131343066636530393534
+          3262646436663034633131316438633230383330633533350a386365313239303464383636376338
+          61656662333939333063343131633963636431663136643137636664633233633133396339613861
+          3038613063626138610a333566393937643630366564653163613364323965396130613433316537
+          39653335393831633362343934363866346262613166393561666336623062393935

From 2ecf6f175b8dd633cfd172cac645d58bc052198f Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 13 Oct 2022 10:49:51 +0100
Subject: [PATCH 18/21] ci-multinode: Use insecure Docker registry

Default changed in Xena
---
 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index 60c5eef5c..8d0991f88 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -5,6 +5,11 @@
 # Docker namespace to use for Kolla images. Default is 'kolla'.
 kolla_docker_namespace: stackhpc-dev
 
+# Whether docker should be configured to use an insecure registry for Kolla
+# # images. Default is false, unless docker_registry_enabled is true and
+# # docker_registry_enable_tls is false.
+kolla_docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}"
+
 ###############################################################################
 # Network configuration.
 

From 89e5f87dca4d630ec61ee7cdbc4f975840d485cd Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzk@stackhpc.com>
Date: Wed, 12 Oct 2022 11:13:54 +0200
Subject: [PATCH 19/21] Removing unnecessary config parts. Compute hosts are
 BM's

---
 etc/kayobe/environments/ci-multinode/kolla/globals.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
index ebfe64897..e7afedf68 100644
--- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml
+++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml
@@ -1,14 +1,4 @@
 ---
-# Most development environments will use nested virtualisation, and we can't
-# guarantee that nested KVM support is available. Use QEMU as a lowest common
-# denominator.
-nova_compute_virt_type: qemu
-
-# Reduce the control plane's memory footprint by limiting the number of worker
-# processes to one per-service.
-openstack_service_workers: "1"
-openstack_service_rpc_workers: "1"
-
 # Glance Ceph configuration
 glance_backend_ceph: "yes"
 

From e0a8888e91d9ca6f86b8c49a77d6fa5e2cac77e2 Mon Sep 17 00:00:00 2001
From: Grzegorz Koper <grzegorzk@stackhpc.com>
Date: Wed, 12 Oct 2022 11:17:53 +0200
Subject: [PATCH 20/21] Matching with ci-builder env

---
 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
index 8d0991f88..9cfccab43 100644
--- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml
@@ -32,13 +32,15 @@ stackhpc_repo_epel_version: "{{ stackhpc_pulp_repo_epel_version }}"
 stackhpc_repo_epel_modular_version: "{{ stackhpc_pulp_repo_epel_modular_version }}"
 stackhpc_repo_docker_version: "{{ stackhpc_pulp_repo_docker_version }}"
 stackhpc_repo_centos_stream_advanced_virtualization_version: "{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}"
+stackhpc_repo_centos_stream_nfv_extras_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_extras_version }}"
 stackhpc_repo_centos_stream_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_openstack_wallaby_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_wallaby_version }}"
+stackhpc_repo_centos_stream_openstack_xena_version: "{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}"
+stackhpc_repo_centos_stream_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_8_opstools_version }}"
 stackhpc_repo_centos_stream_powertools_version: "{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}"
-stackhpc_repo_centos_opstools_version: "{{ stackhpc_pulp_repo_centos_8_opstools_version }}"
 stackhpc_repo_centos_stream_storage_ceph_pacific_version: "{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}"
 stackhpc_repo_elasticsearch_logstash_kibana_7_x_version: "{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}"
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
+stackhpc_repo_mariadb_10_5_centos8_version: "{{ stackhpc_pulp_repo_mariadb_10_5_centos8_version }}"
 stackhpc_repo_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
 stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_pulp_repo_treasuredata_4_version }}"
@@ -47,7 +49,6 @@ stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version
 stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
 stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
 
-
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"

From 4ab17622a7ed1399040102326232f810843822e5 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 13 Oct 2022 11:01:44 +0100
Subject: [PATCH 21/21] ci-multinode: fix pep8 issues

---
 etc/kayobe/environments/ci-multinode/cephadm.yml     | 1 -
 etc/kayobe/environments/ci-multinode/compute.yml     | 3 +++
 etc/kayobe/environments/ci-multinode/controllers.yml | 3 +++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml
index 812f89a06..b4cb2ca0d 100644
--- a/etc/kayobe/environments/ci-multinode/cephadm.yml
+++ b/etc/kayobe/environments/ci-multinode/cephadm.yml
@@ -59,4 +59,3 @@ cephadm_keys:
       osd: "profile rbd pool=images"
       mgr: "profile rbd pool=images"
     state: present
-
diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml
index 254afc8e4..74e7ef86c 100644
--- a/etc/kayobe/environments/ci-multinode/compute.yml
+++ b/etc/kayobe/environments/ci-multinode/compute.yml
@@ -1,2 +1,5 @@
+---
+# List of storage volume groups. See mrlesmithjr.manage-lvm role for
+# format.
 compute_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"
diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml
index 9198a4d4c..3f612ecd8 100644
--- a/etc/kayobe/environments/ci-multinode/controllers.yml
+++ b/etc/kayobe/environments/ci-multinode/controllers.yml
@@ -1,2 +1,5 @@
+---
+# List of storage volume groups. See mrlesmithjr.manage-lvm role for
+# format.
 controller_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"