From c3e143841f19b92a5e1e6f6f10ce58533ca1517b Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 14 Oct 2022 13:21:22 +0000
Subject: [PATCH 01/13] Adds Rocky 8 pulp repositories to dnf.yml (#152)

* Adds Rocky 8 pulp repositories to dnf.yml

This configures the hosts to use the mirrors in the local
pulp. Prior to this change kayobe would attempt to configure
CentOS stream mirrors on a Rocky host.

* Use facts to pick dnf repositories
---
 etc/kayobe/dnf.yml      | 53 +++++++++++++++++++++++++++++++----------
 etc/kayobe/stackhpc.yml | 12 ++++++++++
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index 359c9cefc..c63276a28 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -41,7 +41,27 @@
 #     file: myrepo
 #     gpgkey: http://gpgkey
 #     gpgcheck: yes
-dnf_custom_repos:
+dnf_custom_repos: "{{ dnf_custom_repos_el8 | combine(lookup('vars', 'dnf_custom_repos_' ~ ansible_facts.distribution | lower )) }}"
+
+# Custom repositories shared between all RHEL derivatives.
+dnf_custom_repos_el8:
+  epel:
+    baseurl: "{{ stackhpc_repo_epel_url }}"
+    description: "Extra Packages for Enterprise Linux $releasever - $basearch"
+    enabled: "{{ dnf_enable_epel | bool }}"
+    file: epel
+    gpgkey: "{{ dnf_epel_gpg_key_url }}"
+    gpgcheck: yes
+  epel-modular:
+    baseurl: "{{ stackhpc_repo_epel_modular_url }}"
+    description: "Extra Packages for Enterprise Linux Modular $releasever - $basearch"
+    enabled: "{{ dnf_enable_epel | bool }}"
+    file: epel-modular
+    gpgkey: "{{ dnf_epel_gpg_key_url }}"
+    gpgcheck: yes
+
+# CentOS Stream 8 specific repositories
+dnf_custom_repos_centos:
   appstream:
     baseurl: "{{ stackhpc_repo_centos_stream_appstream_url }}"
     description: "CentOS Stream $releasever - AppStream"
@@ -60,19 +80,26 @@ dnf_custom_repos:
     file: CentOS-Stream-Extras
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
     gpgcheck: yes
-  epel:
-    baseurl: "{{ stackhpc_repo_epel_url }}"
-    description: "Extra Packages for Enterprise Linux $releasever - $basearch"
-    enabled: "{{ dnf_enable_epel | bool }}"
-    file: epel
-    gpgkey: "{{ dnf_epel_gpg_key_url }}"
+
+# Rocky 8 specific repositories
+dnf_custom_repos_rocky:
+  appstream:
+    baseurl: "{{ stackhpc_repo_rocky_appstream_url }}"
+    description: "Rocky Linux $releasever - AppStream"
+    file: Rocky-AppStream
+    gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
     gpgcheck: yes
-  epel-modular:
-    baseurl: "{{ stackhpc_repo_epel_modular_url }}"
-    description: "Extra Packages for Enterprise Linux Modular $releasever - $basearch"
-    enabled: "{{ dnf_enable_epel | bool }}"
-    file: epel-modular
-    gpgkey: "{{ dnf_epel_gpg_key_url }}"
+  baseos:
+    baseurl: "{{ stackhpc_repo_rocky_baseos_url }}"
+    description: "Rocky Linux $releasever - BaseOS"
+    file: Rocky-BaseOS
+    gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
+    gpgcheck: yes
+  extras:
+    baseurl: "{{ stackhpc_repo_rocky_extras_url }}"
+    description: "Rocky Linux $releasever - Extras"
+    file: Rocky-Extras
+    gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
     gpgcheck: yes
 
 # Whether to enable EPEL repositories. This affects RedHat-based systems only.
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 6da0ba3ee..2846b4aec 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -95,6 +95,18 @@ stackhpc_repo_rabbitmq_server_version: "{{ stackhpc_repo_distribution }}"
 stackhpc_repo_treasuredata_4_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/treasuredata/4/redhat/8/x86_64/{{ stackhpc_repo_treasuredata_4_version }}"
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_repo_distribution }}"
 
+# Rocky 8 BaseOS
+stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}"
+stackhpc_repo_rocky_baseos_version: "{{ stackhpc_repo_distribution }}"
+
+# Rocky 8 AppStream
+stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}"
+stackhpc_repo_rocky_appstream_version: "{{ stackhpc_repo_distribution }}"
+
+# Rocky 8 extras
+stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}"
+stackhpc_repo_rocky_extras_version: "{{ stackhpc_repo_distribution }}"
+
 ###############################################################################
 # Sources
 

From abdb0fb072658910869c7b4ced063fc6dbf6c931 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Wed, 31 Aug 2022 20:11:52 +0100
Subject: [PATCH 02/13] Adds Rocky 8 repository config for local pulp (#146)

* Adds Rocky 8 repository config for local pulp

We support Rocky 8 as a host OS with os_distribution == 'rocky'. This
adds the configuration necessary to use the RPM snapshots from ark.

* Abstract over version number v1

Unsure if it would be better to:

1) Update the remote on the repository
2) Change the publication to point to a new repository and keep
repositories for every minor version

Advantages of 1:

- Single repository in pulp

Advantages of 2:

- Less chance of conflicts (although I think an old repo is just an old snapshot)

* Fixups

* Address code review comment

Upadting variable name after feedback
---
 etc/kayobe/pulp-repo-versions.yml |   5 ++
 etc/kayobe/pulp.yml               | 129 +++++++++++++++++++++++++++---
 2 files changed, 125 insertions(+), 9 deletions(-)

diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml
index dba18e296..4864866bd 100644
--- a/etc/kayobe/pulp-repo-versions.yml
+++ b/etc/kayobe/pulp-repo-versions.yml
@@ -21,3 +21,8 @@ stackhpc_pulp_repo_treasuredata_4_version: 20220823T042339
 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20220804T040153
 stackhpc_pulp_repo_ubuntu_focal_security_version: 20220804T040153
 stackhpc_pulp_repo_ubuntu_focal_version: 20220804T040153
+stackhpc_pulp_repo_rocky_8_6_appstream_version: "20220606T111205"
+stackhpc_pulp_repo_rocky_8_6_baseos_version: "20220606T111205"
+stackhpc_pulp_repo_rocky_8_6_extras_version: "20220606T111205"
+stackhpc_pulp_repo_rocky_8_6_nfv_version: "20220606T111205"
+stackhpc_pulp_repo_rocky_8_6_powertools_version: "20220606T111205"
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 9c0545c39..24d12dc42 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -167,6 +167,21 @@ stackhpc_pulp_distribution_deb_production:
 # Whether to sync CentOS Stream 8 packages.
 stackhpc_pulp_sync_centos_stream8: "{{ os_distribution == 'centos' }}"
 
+# Whether to sync Rocky Linux 8 packages.
+stackhpc_pulp_sync_rocky_8: "{{ os_distribution == 'rocky' }}"
+# Rocky 8 minor version number. Supported values: 6.
+stackhpc_pulp_repo_rocky_8_minor_version: 6
+# Rocky 8 Snapshot versions. The defaults use the appropriate version from
+# pulp-repo-versions.yml for the selected minor release.
+stackhpc_pulp_repo_rocky_8_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_appstream_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}"
+stackhpc_pulp_repo_rocky_8_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_baseos_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}"
+stackhpc_pulp_repo_rocky_8_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_extras_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}"
+stackhpc_pulp_repo_rocky_8_nfv_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_nfv_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}"
+stackhpc_pulp_repo_rocky_8_powertools_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_8_%s_powertools_version' % stackhpc_pulp_repo_rocky_8_minor_version) }}"
+
+# Whether to sync packages common to all RHEL derivatives.
+stackhpc_pulp_sync_el_8: "{{ stackhpc_pulp_sync_rocky_8 or stackhpc_pulp_sync_centos_stream8 }}"
+
 stackhpc_pulp_repository_rpm_repos:
   # Base CentOS 8 Stream repositories
   - name: CentOS Stream 8 - AppStream
@@ -194,6 +209,48 @@ stackhpc_pulp_repository_rpm_repos:
     state: present
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
 
+  # Base Rocky 8 repositories
+  - name: Rocky Linux 8 - AppStream
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}"
+    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
+    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: Rocky Linux 8 - BaseOS
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}"
+    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
+    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: Rocky Linux 8 - Extras
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}"
+    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
+    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: Rocky Linux 8 - NFV
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}"
+    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
+    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: Rocky Linux 8 - PowerTools
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_6_powertools_version }}"
+    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
+    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+
   # EPEL repositories
   - name: Extra Packages for Enterprise Linux 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}"
@@ -202,7 +259,7 @@ stackhpc_pulp_repository_rpm_repos:
     policy: on_demand
     sync_policy: mirror_content_only
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: Extra Packages for Enterprise Linux Modular 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}"
     client_cert: "{{ stackhpc_release_pulp_client_cert }}"
@@ -210,7 +267,7 @@ stackhpc_pulp_repository_rpm_repos:
     policy: on_demand
     sync_policy: mirror_complete
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
   # Third-party repositories
   - name: Docker CE for CentOS 8
@@ -220,7 +277,7 @@ stackhpc_pulp_repository_rpm_repos:
     policy: on_demand
     sync_policy: mirror_complete
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 }}"
 
 # Publication format is a subset of distribution.
 stackhpc_pulp_publication_rpm_development: "{{ stackhpc_pulp_distribution_rpm_development }}"
@@ -247,24 +304,51 @@ stackhpc_pulp_distribution_rpm_development:
     state: present
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
 
+  # Base Rocky 8 repositories
+  - name: "rocky-8-appstream-development"
+    repository: Rocky Linux 8 - AppStream
+    base_path: "rocky/8/AppStream/x86_64/os/development"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-baseos-development"
+    repository: Rocky Linux 8 - BaseOS
+    base_path: "rocky/8/BaseOS/x86_64/os/development"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-extras-development"
+    repository: Rocky Linux 8 - Extras
+    base_path: "rocky/8/extras/x86_64/os/development"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-nfv-development"
+    repository: Rocky Linux 8 - NFV
+    base_path: "rocky/8/nfv/x86_64/os/development"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-powertools-development"
+    repository: Rocky Linux 8 - PowerTools
+    base_path: "rocky/8/PowerTools/x86_64/os/development"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+
   # EPEL repositories
   - name: "extra-packages-for-enterprise-linux-8-x86_64-development"
     repository: Extra Packages for Enterprise Linux 8 - x86_64
     base_path: "epel/8/Everything/x86_64/development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: "extra-packages-for-enterprise-linux-modular-8-x86_64-development"
     repository: Extra Packages for Enterprise Linux Modular 8 - x86_64
     base_path: "epel/8/Modular/x86_64/development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
   # Third-party repositories
   - name: "docker-ce-for-centos-8-development"
     repository: Docker CE for CentOS 8
     base_path: "docker-ce/centos/8/x86_64/stable/development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
 # Development gets promoted to production.
 stackhpc_pulp_distribution_rpm_production:
@@ -285,24 +369,51 @@ stackhpc_pulp_distribution_rpm_production:
     state: present
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
 
+  # Base Rocky 8 repositories
+  - name: "rocky-8-appstream-production"
+    distribution: "rocky-8-appstream-development"
+    base_path: "rocky/8/AppStream/x86_64/os/production"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-baseos-production"
+    distribution: "rocky-8-baseos-development"
+    base_path: "rocky/8/BaseOS/x86_64/os/production"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-extras-production"
+    distribution: "rocky-8-extras-development"
+    base_path: "rocky/8/extras/x86_64/os/production"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-nfv-production"
+    distribution: "rocky-8-nfv-development"
+    base_path: "rocky/8/nfv/x86_64/os/production"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+  - name: "rocky-8-powertools-production"
+    distribution: "rocky-8-powertools-development"
+    base_path: "rocky/8/PowerTools/x86_64/os/production"
+    state: present
+    required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
+
   # EPEL repositories
   - name: "extra-packages-for-enterprise-linux-8-x86_64-production"
     base_path: "epel/8/Everything/x86_64/production"
     distribution: "extra-packages-for-enterprise-linux-8-x86_64-development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: "extra-packages-for-enterprise-linux-modular-8-x86_64-production"
     base_path: "epel/8/Modular/x86_64/production"
     distribution: "extra-packages-for-enterprise-linux-modular-8-x86_64-development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
   # Third-party repositories
   - name: "docker-ce-for-centos-8-production"
     base_path: "docker-ce/centos/8/x86_64/stable/production"
     distribution: "docker-ce-for-centos-8-development"
     state: present
-    required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
+    required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
 ###############################################################################
 # Containers

From 549935017abf91dde1e4152a03d426ba04d74523 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Thu, 1 Sep 2022 10:56:15 +0100
Subject: [PATCH 03/13] Update docker baseurl to work with Rocky 8 (#153)

This is the equivalent change of:

https://review.opendev.org/c/openstack/ansible-collection-kolla/+/831642
---
 etc/kayobe/kolla/globals.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
index 264c0e663..84cd97511 100644
--- a/etc/kayobe/kolla/globals.yml
+++ b/etc/kayobe/kolla/globals.yml
@@ -1,7 +1,7 @@
 # yamllint disable-file
 ---
 docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}"
-docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg"
+docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg"
 
 {% if kolla_base_distro == 'centos' %}
 bifrost_tag: wallaby-20220921T100954

From f021ffb9edaf01fe92ee2f578cbb73fd27539224 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 14 Oct 2022 13:21:22 +0000
Subject: [PATCH 04/13] Adds a Rocky 8 AIO test

---
 .github/workflows/stackhpc-all-in-one.yml     | 30 +++++-
 .github/workflows/stackhpc-pull-request.yml   | 40 +++++++-
 .../ansible/configure-aio-resources.yml       | 91 ++++++++++---------
 etc/kayobe/compute.yml                        |  6 ++
 etc/kayobe/environments/ci-aio/compute.yml    |  3 +
 .../environments/ci-aio/kolla/globals.yml     |  2 +-
 .../environments/ci-aio/stackhpc-ci.yml       |  4 +-
 etc/kayobe/kolla.yml                          |  1 +
 etc/kayobe/stackhpc.yml                       |  6 +-
 terraform/aio/outputs.tf                      |  2 +-
 .../aio/scripts/configure-local-networking.sh |  3 -
 terraform/aio/templates/userdata.cfg.tpl      |  8 ++
 terraform/aio/vm.tf                           | 32 ++++---
 13 files changed, 156 insertions(+), 72 deletions(-)
 create mode 100644 etc/kayobe/environments/ci-aio/compute.yml

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 4c0a62f45..feb238b66 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -11,14 +11,26 @@ on:
         description: Kayobe container image
         type: string
         required: true
+      os_distribution:
+        description: Host OS distribution
+        type: string
+        default: centos
       neutron_plugin:
         description: Neutron ML2 plugin
         type: string
         required: true
+      vm_user:
+        description: User to use when bootstraping the VM. Leave unset to use os_distribution.
+        type: string
+        default: ''
       vm_image:
         description: Image for the all-in-one VM
         type: string
         default: CentOS-stream8
+      vm_interface:
+        description: Default network interface name
+        type: string
+        default: eth0
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
@@ -56,6 +68,12 @@ jobs:
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
+      - name: Set bootstrap user output
+        id: bootstrap_user
+        run: |
+          user=${{ inputs.vm_user }}
+          echo "::set-output name=value::${user:-"${{ inputs.os_distribution }}"}"
+
       - uses: actions/checkout@v2
         with:
            submodules: true
@@ -81,11 +99,12 @@ jobs:
       - name: Generate terraform.tfvars
         run: |
           cat << EOF > terraform.tfvars
-          ssh_private_key = "id_rsa"
           ssh_public_key = "id_rsa.pub"
+          ssh_private_key = "id_rsa"
+          aio_vm_user = "${{ env.VM_USER }}"
+          aio_vm_interface = "${{ env.VM_INTERFACE }}"
           aio_vm_name = "${{ env.VM_NAME }}"
           aio_vm_image = "${{ env.VM_IMAGE }}"
-          aio_vm_keypair = "${{ env.VM_KEYPAIR }}"
           aio_vm_flavor = "${{ env.VM_FLAVOR }}"
           aio_vm_network = "${{ env.VM_NETWORK }}"
           aio_vm_subnet = "${{ env.VM_SUBNET }}"
@@ -94,10 +113,11 @@ jobs:
         env:
           VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
           VM_IMAGE: ${{ inputs.vm_image }}
-          VM_KEYPAIR: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
           VM_FLAVOR: ${{ inputs.vm_flavor }}
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
+          VM_USER: ${{ steps.bootstrap_user.outputs.value }}
+          VM_INTERFACE: ${{ inputs.vm_interface }}
 
       - name: Terraform Plan
         run: terraform plan
@@ -151,10 +171,14 @@ jobs:
         run: |
           cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/zz-aio-scenario.yml
           ---
+          os_distribution: ${{ env.OS_DISTRIBUTION }}
           kolla_enable_ovn: ${{ env.ENABLE_OVN }}
+          bootstrap_user: ${{ env.BOOTSTRAP_USER }}
           EOF
         env:
           ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
+          OS_DISTRIBUTION: ${{ inputs.os_distribution }}
+          BOOTSTRAP_USER: ${{ steps.bootstrap_user.outputs.value }}
 
       # https://renehernandez.io/snippets/multiline-strings-as-a-job-output-in-github-actions/
       - name: Set SSH key output
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index da4760a84..f19585874 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -35,8 +35,8 @@ jobs:
     uses: ./.github/workflows/stackhpc-build-kayobe-image.yml
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
-  all-in-one-ovs:
-    name: aio (OVS)
+  all-in-one-centos-ovs:
+    name: aio (CentOS OVS)
     needs:
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-all-in-one.yml
@@ -47,8 +47,8 @@ jobs:
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
-  all-in-one-ovn:
-    name: aio (OVN)
+  all-in-one-centos-ovn:
+    name: aio (CentOS OVN)
     needs:
       - build-kayobe-image
     uses: ./.github/workflows/stackhpc-all-in-one.yml
@@ -58,3 +58,35 @@ jobs:
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-rocky-ovs:
+    name: aio (Rocky OVS)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      neutron_plugin: ovs
+      vm_image: Rocky8
+      vm_user: cloud-user
+      vm_interface: ens3
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+
+  all-in-one-rocky-ovn:
+    name: aio (Rocky OVN)
+    needs:
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      neutron_plugin: ovn
+      vm_image: Rocky8
+      vm_user: cloud-user
+      vm_interface: ens3
+      OS_CLOUD: sms-lab-release
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
diff --git a/etc/kayobe/ansible/configure-aio-resources.yml b/etc/kayobe/ansible/configure-aio-resources.yml
index 3fdb88d0c..78450eaeb 100644
--- a/etc/kayobe/ansible/configure-aio-resources.yml
+++ b/etc/kayobe/ansible/configure-aio-resources.yml
@@ -6,47 +6,56 @@
   vars:
     venv: '{{ virtualenv_path }}/openstack'
   tasks:
-    - block:
-        - name: Install python openstack client
-          pip:
-            name: python-openstackclient
-            virtualenv: '{{ venv }}'
-            extra_args: -c "{{ pip_upper_constraints_file }}"
+    - name: Install python build dependencies
+      package:
+        name:
+          - "{% if ansible_facts.os_family == 'RedHat' %}gcc{% else %}build-essential{% endif %}"
+          - "python3-dev{% if ansible_facts.os_family == 'RedHat' %}el{% endif %}"
+        cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}"
+        update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}"
+        state: present
+      become: true
 
-        - name: Include kolla secrets
-          include_vars:
-            dir: '{{ kayobe_env_config_path }}/kolla/'
-            files_matching: passwords.yml
-            name: kolla_passwords
+    - name: Install python openstack client
+      pip:
+        name: python-openstackclient
+        virtualenv: '{{ venv }}'
+        extra_args: -c "{{ pip_upper_constraints_file }}"
 
-        - name: Add an IP to connect to the instances
-          # FIXME: host configure will have bounced the bridge
-          # and removed the IP
-          command: ip a add 10.0.2.1/24 dev breth1
-          register: result
-          failed_when: 'result.rc !=0 and "RTNETLINK answers: File exists" not in
-            result.stderr'
-          changed_when: result.rc == 0
-          become: true
+    - name: Include kolla secrets
+      include_vars:
+        dir: '{{ kayobe_env_config_path }}/kolla/'
+        files_matching: passwords.yml
+        name: kolla_passwords
 
-        - name: Run init-run-once
-          script:
-            cmd: scripts/aio-init.sh
-            creates: /tmp/.init-runonce
-          environment:
-            KOLLA_OPENSTACK_COMMAND: '{{ venv }}/bin/openstack'
-            OS_PROJECT_DOMAIN_NAME: Default
-            OS_USER_DOMAIN_NAME: Default
-            OS_PROJECT_NAME: admin
-            OS_TENANT_NAME: admin
-            OS_USERNAME: admin
-            OS_PASSWORD: "{{ kolla_passwords.keystone_admin_password | mandatory('Could\
-              \ not find keystone_admin_password in passwords.yml') }}"
-            # Use kolla_external_fqdn in wallaby
-            OS_AUTH_URL: http://{{ kolla_external_fqdn | default(public_net_name
-              | net_fqdn) | default(public_net_name | net_vip_address, true) }}:5000
-            OS_INTERFACE: public
-            OS_ENDPOINT_TYPE: publicURL
-            OS_IDENTITY_API_VERSION: 3
-            OS_REGION_NAME: RegionOne
-            OS_AUTH_PLUGIN: password
+    - name: Add an IP to connect to the instances
+      # FIXME: host configure will have bounced the bridge
+      # and removed the IP
+      command: ip a add 10.0.2.1/24 dev breth1
+      register: result
+      failed_when: 'result.rc !=0 and "RTNETLINK answers: File exists" not in
+        result.stderr'
+      changed_when: result.rc == 0
+      become: true
+
+    - name: Run init-run-once
+      script:
+        cmd: scripts/aio-init.sh
+        creates: /tmp/.init-runonce
+      environment:
+        KOLLA_OPENSTACK_COMMAND: '{{ venv }}/bin/openstack'
+        OS_PROJECT_DOMAIN_NAME: Default
+        OS_USER_DOMAIN_NAME: Default
+        OS_PROJECT_NAME: admin
+        OS_TENANT_NAME: admin
+        OS_USERNAME: admin
+        OS_PASSWORD: "{{ kolla_passwords.keystone_admin_password | mandatory('Could\
+          \ not find keystone_admin_password in passwords.yml') }}"
+        # Use kolla_external_fqdn in wallaby
+        OS_AUTH_URL: http://{{ kolla_external_fqdn | default(public_net_name
+          | net_fqdn) | default(public_net_name | net_vip_address, true) }}:5000
+        OS_INTERFACE: public
+        OS_ENDPOINT_TYPE: publicURL
+        OS_IDENTITY_API_VERSION: 3
+        OS_REGION_NAME: RegionOne
+        OS_AUTH_PLUGIN: password
diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml
index 893aaa2e3..e70d8b53e 100644
--- a/etc/kayobe/compute.yml
+++ b/etc/kayobe/compute.yml
@@ -172,6 +172,12 @@
 # compute_qemu_conf_extra.
 #compute_qemu_conf:
 
+# Whether to enable libvirt SASL authentication. Default is true.
+#compute_libvirt_enable_sasl:
+
+# libvirt SASL password. Default is unset.
+#compute_libvirt_sasl_password:
+
 # Whether to enable a libvirt TLS listener. Default is false.
 #compute_libvirt_enable_tls:
 
diff --git a/etc/kayobe/environments/ci-aio/compute.yml b/etc/kayobe/environments/ci-aio/compute.yml
new file mode 100644
index 000000000..e5bd95d6d
--- /dev/null
+++ b/etc/kayobe/environments/ci-aio/compute.yml
@@ -0,0 +1,3 @@
+---
+
+compute_libvirt_sasl_password: "{{ lookup('password', kayobe_env_config_path ~ '/secrets/libvirt_sasl_password') }}"
diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml
index 8bbef4a78..4ef594ec5 100644
--- a/etc/kayobe/environments/ci-aio/kolla/globals.yml
+++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml
@@ -10,4 +10,4 @@ openstack_service_workers: "1"
 openstack_service_rpc_workers: "1"
 
 docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}"
-docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg"
+docker_yum_gpgkey: "https://download.docker.com/linux/centos/gpg"
diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
index 8e9ea16e5..7607ce3d1 100644
--- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
@@ -41,7 +41,9 @@ stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_
 stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
 stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
 stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
-
+stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_6_baseos_version }}"
+stackhpc_repo_rocky_appstream_version: "{{ stackhpc_pulp_repo_rocky_8_6_appstream_version }}"
+stackhpc_repo_rocky_extras_version: "{{ stackhpc_pulp_repo_rocky_8_6_extras_version }}"
 
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index ff7cb0f05..286f48172 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -522,6 +522,7 @@ kolla_build_customizations: "{{ kolla_build_customizations_common | combine(koll
 #kolla_enable_keystone_horizon_policy_file:
 #kolla_enable_kibana:
 #kolla_enable_kuryr:
+kolla_enable_nova_libvirt_container: "{{ os_distribution not in ['rocky'] }}"
 #kolla_enable_magnum:
 #kolla_enable_manila:
 #kolla_enable_manila_backend_cephfs_native:
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 2846b4aec..03aa3071a 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -96,15 +96,15 @@ stackhpc_repo_treasuredata_4_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/t
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 BaseOS
-stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}"
+stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}"
 stackhpc_repo_rocky_baseos_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 AppStream
-stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}"
+stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}"
 stackhpc_repo_rocky_appstream_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 extras
-stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}"
+stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}"
 stackhpc_repo_rocky_extras_version: "{{ stackhpc_repo_distribution }}"
 
 ###############################################################################
diff --git a/terraform/aio/outputs.tf b/terraform/aio/outputs.tf
index faa49b079..8d184e3a9 100644
--- a/terraform/aio/outputs.tf
+++ b/terraform/aio/outputs.tf
@@ -11,5 +11,5 @@ output "access_gw" {
 }
 
 output "access_interface" {
-  value = "eth0"
+  value = var.aio_vm_interface
 }
diff --git a/terraform/aio/scripts/configure-local-networking.sh b/terraform/aio/scripts/configure-local-networking.sh
index c7bd70bf8..d8f7419aa 100644
--- a/terraform/aio/scripts/configure-local-networking.sh
+++ b/terraform/aio/scripts/configure-local-networking.sh
@@ -53,6 +53,3 @@ done
 if ! sudo ip a show dev breth1 | grep $public_ip/24 >/dev/null 2>&1; then
   sudo ip a add $public_ip/24 dev breth1
 fi
-
-# This prevents network.service from restarting correctly.
-sudo killall dhclient || true
diff --git a/terraform/aio/templates/userdata.cfg.tpl b/terraform/aio/templates/userdata.cfg.tpl
index e94bafd3c..9ddca24d9 100644
--- a/terraform/aio/templates/userdata.cfg.tpl
+++ b/terraform/aio/templates/userdata.cfg.tpl
@@ -2,3 +2,11 @@
 # Don't automatically mount ephemeral disk
 mounts:
   - [/dev/vdb, null]
+# WORKAROUND: internal DNS missing from SMS lab.
+runcmd:
+  - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts'
+  - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts'
+# Configure SSH keys here, to avoid creating an ephemeral keypair.
+# This means only the instance needs to be cleaned up if the destroy fails.
+ssh_authorized_keys:
+ - ${ssh_public_key}
diff --git a/terraform/aio/vm.tf b/terraform/aio/vm.tf
index 9b9ccf631..cf071f1a1 100644
--- a/terraform/aio/vm.tf
+++ b/terraform/aio/vm.tf
@@ -1,8 +1,8 @@
-variable "ssh_private_key" {
+variable "ssh_public_key" {
   type = string
 }
 
-variable "ssh_public_key" {
+variable "ssh_private_key" {
   type = string
 }
 
@@ -16,8 +16,13 @@ variable "aio_vm_image" {
   default = "CentOS-stream8"
 }
 
-variable "aio_vm_keypair" {
+variable "aio_vm_user" {
+  type = string
+}
+
+variable "aio_vm_interface" {
   type = string
+  default = "eth0"
 }
 
 variable "aio_vm_flavor" {
@@ -41,17 +46,11 @@ data "openstack_networking_subnet_v2" "network" {
   name = var.aio_vm_subnet
 }
 
-resource "openstack_compute_keypair_v2" "keypair" {
-  name = var.aio_vm_keypair
-  public_key = file(var.ssh_public_key)
-}
-
 resource "openstack_compute_instance_v2" "kayobe-aio" {
   name         = var.aio_vm_name
   flavor_name  = var.aio_vm_flavor
-  key_pair     = var.aio_vm_keypair
   config_drive = true
-  user_data    = file("templates/userdata.cfg.tpl")
+  user_data    = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)})
   network {
     name = var.aio_vm_network
   }
@@ -59,7 +58,7 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
   block_device {
     uuid                  = data.openstack_images_image_v2.image.id
     source_type           = "image"
-    volume_size           = 100
+    volume_size           = 200
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
@@ -67,27 +66,30 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
 
   provisioner "file" {
     source      = "scripts/configure-local-networking.sh"
-    destination = "/home/centos/configure-local-networking.sh"
+    destination = "/home/${var.aio_vm_user}/configure-local-networking.sh"
 
     connection {
       type        = "ssh"
       host        = self.access_ip_v4
-      user        = "centos"
+      user        = var.aio_vm_user
       private_key = file(var.ssh_private_key)
     }
   }
 
   provisioner "remote-exec" {
     inline = [
-      "sudo bash /home/centos/configure-local-networking.sh"
+      "sudo bash /home/${var.aio_vm_user}/configure-local-networking.sh"
     ]
 
     connection {
       type        = "ssh"
       host        = self.access_ip_v4
-      user        = "centos"
+      user        = var.aio_vm_user
       private_key = file(var.ssh_private_key)
+      # /tmp is noexec when using stackhpc LVM layout
+      script_path = "/home/${var.aio_vm_user}/.configure-local-networking"
     }
 
   }
+
 }

From 0fc5883dca043b1e1e5f7ab1b5469892150bbf33 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 14 Oct 2022 16:52:57 +0100
Subject: [PATCH 05/13] Use ansible fork with Rocky 8 support

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 73ef04b58..c6a9bc335 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-git+https://github.com/stackhpc/kayobe@stackhpc/wallaby
+kayobe@git+https://github.com/stackhpc/kayobe@feature/wallaby/rocky-8

From a7e1c7d26c6307e38b9cee1a99ea9459deaefdc1 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Fri, 28 Oct 2022 10:41:57 +0100
Subject: [PATCH 06/13] Install fork of ansible unconditionally

The kayobe docker image fron kayobe-automation was being built
without Rocky Linux support. This is because os_distribution was
not being set add build time.
---
 .automation          | 2 +-
 etc/kayobe/kolla.yml | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.automation b/.automation
index a5749321e..d4aa284af 160000
--- a/.automation
+++ b/.automation
@@ -1 +1 @@
-Subproject commit a5749321ebcaa9798f1d63d7dc15f1380540f0a3
+Subproject commit d4aa284afe40c882639637d0f50ebe927439d681
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 286f48172..137e07d38 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -48,11 +48,10 @@ kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}"
 #kolla_ansible_venv:
 
 # Extra requirements to install inside the kolla-ansible virtualenv.
-kolla_ansible_venv_extra_requirements: "{{ lookup('vars', 'kolla_ansible_venv_extra_requirements_' ~ os_distribution, default=[]) }}"
-
-# Rocky specific requirements in the kolla-ansible virtualenv
-kolla_ansible_venv_extra_requirements_rocky:
-  # NOTE(wszumski): This is wallaby specific as we can use ansible-core 2.11 in Xena.
+kolla_ansible_venv_extra_requirements:
+  # NOTE(wszumski): Fork with support for Rocky Linux target hosts. Remove in
+  # Xena as kayobe supports ansible 2.11 which has Rocky Support out of the
+  # box.
   - 'ansible-base@git+https://github.com/stackhpc/ansible@stackhpc/2.10/rocky'
 
 # Pip requirement specifier for the ansible package. NOTE: This limits the

From f49143f451054fa4367fdd5c640fdd7eb7c543f3 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Tue, 1 Nov 2022 17:18:07 +0000
Subject: [PATCH 07/13] Workaround lack of ovs-vsctl on the host.

---
 terraform/aio/templates/userdata.cfg.tpl | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/terraform/aio/templates/userdata.cfg.tpl b/terraform/aio/templates/userdata.cfg.tpl
index 9ddca24d9..9cae72983 100644
--- a/terraform/aio/templates/userdata.cfg.tpl
+++ b/terraform/aio/templates/userdata.cfg.tpl
@@ -10,3 +10,12 @@ runcmd:
 # This means only the instance needs to be cleaned up if the destroy fails.
 ssh_authorized_keys:
  - ${ssh_public_key}
+
+write_files:
+  # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409
+  - content: |
+      #!/bin/bash
+      docker exec openvswitch_vswitchd ovs-vsctl "$@"
+    owner: root:root
+    path: /usr/bin/ovs-vsctl
+    permissions: '0755'

From 9de51ff655146571f47d0e7d15484e28ac6a1e31 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Tue, 15 Nov 2022 10:19:26 +0000
Subject: [PATCH 08/13] Switch back to stackhpc/wallaby

The PR has now been merged.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c6a9bc335..c1a219ce3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-kayobe@git+https://github.com/stackhpc/kayobe@feature/wallaby/rocky-8
+kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/wallaby

From 7da5e12bec6982d0c6ae1cc011d3ab2d2430511a Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Tue, 15 Nov 2022 10:34:10 +0000
Subject: [PATCH 09/13] centos user has changed to cloud-user

The images on smslab are now based on centos-minimal.
---
 .github/workflows/stackhpc-pull-request.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index f19585874..6e875c44e 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -43,6 +43,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovs
+      vm_user: cloud-user
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -55,6 +56,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovn
+      vm_user: cloud-user
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'

From e49f48219fcda4ab637a2c71560f8162f3b67541 Mon Sep 17 00:00:00 2001
From: Will Szumski <williamszumski@gmail.com>
Date: Tue, 15 Nov 2022 10:55:42 +0000
Subject: [PATCH 10/13] Add os-distribution to tempest artifact name

This allows you to differentiate the artifacts across the distros.
---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index feb238b66..10ad8623c 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -241,7 +241,7 @@ jobs:
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: tempest-results-${{ inputs.neutron_plugin }}
+          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.neutron_plugin }}
           path: tempest-artifacts/*
 
       - name: Fail if any Tempest tests failed

From 72e96ff7cb4ab560ffbda2e786488117a06926b8 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Thu, 17 Nov 2022 09:52:51 +0000
Subject: [PATCH 11/13] Remove bootstrap override

This is no longer necessary now the centos image also uses cloud-user.
---
 .github/workflows/stackhpc-all-in-one.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 32be38da4..aa19d17f5 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -173,12 +173,10 @@ jobs:
           ---
           os_distribution: ${{ env.OS_DISTRIBUTION }}
           kolla_enable_ovn: ${{ env.ENABLE_OVN }}
-          bootstrap_user: ${{ env.BOOTSTRAP_USER }}
           EOF
         env:
           ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
           OS_DISTRIBUTION: ${{ inputs.os_distribution }}
-          BOOTSTRAP_USER: ${{ steps.bootstrap_user.outputs.value }}
 
       # https://renehernandez.io/snippets/multiline-strings-as-a-job-output-in-github-actions/
       - name: Set SSH key output

From ece66522438d605d515df09bc45b8f2f50cdcff3 Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Thu, 17 Nov 2022 09:53:41 +0000
Subject: [PATCH 12/13] Revert volume_size to 100GB

200G was only necessary for testing the DIB images from kayobe-config
with the default LVM logical volume sizes.
---
 terraform/aio/vm.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/aio/vm.tf b/terraform/aio/vm.tf
index 6ae6c76d6..a7a7e0f07 100644
--- a/terraform/aio/vm.tf
+++ b/terraform/aio/vm.tf
@@ -58,7 +58,7 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
   block_device {
     uuid                  = data.openstack_images_image_v2.image.id
     source_type           = "image"
-    volume_size           = 200
+    volume_size           = 100
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true

From 8b906b3a0c0201396adf9bfce245ec20da20be3e Mon Sep 17 00:00:00 2001
From: Will Szumski <will@stackhpc.com>
Date: Thu, 17 Nov 2022 10:03:44 +0000
Subject: [PATCH 13/13] Remove configure-local-networking

---
 .github/workflows/stackhpc-all-in-one.yml     | 13 -----
 .github/workflows/stackhpc-pull-request.yml   |  4 --
 terraform/aio/README.rst                      |  2 -
 .../aio/scripts/configure-local-networking.sh | 55 -------------------
 terraform/aio/vm.tf                           | 35 ------------
 5 files changed, 109 deletions(-)
 delete mode 100644 terraform/aio/scripts/configure-local-networking.sh

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index aa19d17f5..3bb92adcf 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -19,10 +19,6 @@ on:
         description: Neutron ML2 plugin
         type: string
         required: true
-      vm_user:
-        description: User to use when bootstraping the VM. Leave unset to use os_distribution.
-        type: string
-        default: ''
       vm_image:
         description: Image for the all-in-one VM
         type: string
@@ -68,12 +64,6 @@ jobs:
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
-      - name: Set bootstrap user output
-        id: bootstrap_user
-        run: |
-          user=${{ inputs.vm_user }}
-          echo "::set-output name=value::${user:-"${{ inputs.os_distribution }}"}"
-
       - uses: actions/checkout@v2
         with:
            submodules: true
@@ -100,8 +90,6 @@ jobs:
         run: |
           cat << EOF > terraform.tfvars
           ssh_public_key = "id_rsa.pub"
-          ssh_private_key = "id_rsa"
-          aio_vm_user = "${{ env.VM_USER }}"
           aio_vm_interface = "${{ env.VM_INTERFACE }}"
           aio_vm_name = "${{ env.VM_NAME }}"
           aio_vm_image = "${{ env.VM_IMAGE }}"
@@ -116,7 +104,6 @@ jobs:
           VM_FLAVOR: ${{ inputs.vm_flavor }}
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
-          VM_USER: ${{ steps.bootstrap_user.outputs.value }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
 
       - name: Terraform Plan
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index 6e875c44e..8f9160e34 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -43,7 +43,6 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovs
-      vm_user: cloud-user
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -56,7 +55,6 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovn
-      vm_user: cloud-user
       OS_CLOUD: sms-lab-release
     secrets: inherit
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
@@ -71,7 +69,6 @@ jobs:
       os_distribution: rocky
       neutron_plugin: ovs
       vm_image: Rocky8
-      vm_user: cloud-user
       vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
@@ -87,7 +84,6 @@ jobs:
       os_distribution: rocky
       neutron_plugin: ovn
       vm_image: Rocky8
-      vm_user: cloud-user
       vm_interface: ens3
       OS_CLOUD: sms-lab-release
     secrets: inherit
diff --git a/terraform/aio/README.rst b/terraform/aio/README.rst
index 3646b757d..edb465943 100644
--- a/terraform/aio/README.rst
+++ b/terraform/aio/README.rst
@@ -82,11 +82,9 @@ Generate Terraform variables:
 .. code-block:: console
 
    cat << EOF > terraform.tfvars
-   ssh_private_key = "id_rsa"
    ssh_public_key = "id_rsa.pub"
    aio_vm_name = "kayobe-aio"
    aio_vm_image = "CentOS-stream8"
-   aio_vm_keypair = "kayobe-aio"
    aio_vm_flavor = "general.v1.medium"
    aio_vm_network = "stackhpc-ipv4-geneve"
    aio_vm_subnet = "stackhpc-ipv4-geneve-subnet"
diff --git a/terraform/aio/scripts/configure-local-networking.sh b/terraform/aio/scripts/configure-local-networking.sh
deleted file mode 100644
index d8f7419aa..000000000
--- a/terraform/aio/scripts/configure-local-networking.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# WORKAROUND: internal DNS missing from SMS lab.
-cat << EOF | sudo tee -a /etc/hosts
-10.0.0.34 pelican pelican.service.compute.sms-lab.cloud
-10.205.3.187 pulp-server pulp-server.internal.sms-cloud
-EOF
-
-# IP of the seed hypervisor on the OpenStack 'public' network created by init-runonce.sh.
-public_ip="10.0.2.1"
-
-# IP addresses on the all-in-one Kayobe cloud network.
-# These IP addresses map to those statically configured in
-# etc/kayobe/network-allocation.yml and etc/kayobe/networks.yml.
-controller_vip=192.168.33.2
-
-# Forward the following ports to the controller.
-# 80: Horizon
-# 6080: VNC console
-forwarded_ports="80 6080"
-
-sudo ip l add breth1 type bridge
-sudo ip l set breth1 up
-sudo ip a add 192.168.33.3/24 dev breth1
-sudo ip l add eth1 type dummy
-sudo ip l set eth1 up
-sudo ip l set eth1 master breth1
-
-iface=$(ip route | awk '$1 == "default" {print $5; exit}')
-
-#sudo iptables -A POSTROUTING -t nat -o $iface -j MASQUERADE
-sudo sysctl -w net.ipv4.conf.all.forwarding=1
-
-# Install iptables.
-if $(which dnf >/dev/null 2>&1); then
-    sudo dnf -y install iptables
-fi
-
-# Configure port forwarding from the hypervisor to the Horizon GUI on the
-# controller.
-sudo iptables -A FORWARD -i $iface -o breth1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
-sudo iptables -A FORWARD -i breth1 -o $iface -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
-for port in $forwarded_ports; do
-  # Allow new connections.
-  sudo iptables -A FORWARD -i $iface -o breth1 -p tcp --syn --dport $port -m conntrack --ctstate NEW -j ACCEPT
-  # Destination NAT.
-  sudo iptables -t nat -A PREROUTING -i $iface -p tcp --dport $port -j DNAT --to-destination $controller_vip
-done
-
-# Configure an IP on the 'public' network to allow access to/from the cloud.
-if ! sudo ip a show dev breth1 | grep $public_ip/24 >/dev/null 2>&1; then
-  sudo ip a add $public_ip/24 dev breth1
-fi
diff --git a/terraform/aio/vm.tf b/terraform/aio/vm.tf
index a7a7e0f07..911b0a5a3 100644
--- a/terraform/aio/vm.tf
+++ b/terraform/aio/vm.tf
@@ -2,10 +2,6 @@ variable "ssh_public_key" {
   type = string
 }
 
-variable "ssh_private_key" {
-  type = string
-}
-
 variable "aio_vm_name" {
   type    = string
   default = "kayobe-aio"
@@ -16,10 +12,6 @@ variable "aio_vm_image" {
   default = "CentOS-stream8"
 }
 
-variable "aio_vm_user" {
-  type = string
-}
-
 variable "aio_vm_interface" {
   type = string
   default = "eth0"
@@ -64,31 +56,4 @@ resource "openstack_compute_instance_v2" "kayobe-aio" {
     delete_on_termination = true
   }
 
-  provisioner "file" {
-    source      = "scripts/configure-local-networking.sh"
-    destination = "/home/${var.aio_vm_user}/configure-local-networking.sh"
-    connection {
-      type        = "ssh"
-      host        = self.access_ip_v4
-      user        = var.aio_vm_user
-      private_key = file(var.ssh_private_key)
-    }
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "sudo bash /home/${var.aio_vm_user}/configure-local-networking.sh"
-    ]
-
-    connection {
-      type        = "ssh"
-      host        = self.access_ip_v4
-      user        = var.aio_vm_user
-      private_key = file(var.ssh_private_key)
-      # /tmp is noexec when using stackhpc LVM layout
-      script_path = "/home/${var.aio_vm_user}/.configure-local-networking"
-    }
-
-  }
-
 }