From e10923a699f0e7c717a613745a9c96f27f4e3a01 Mon Sep 17 00:00:00 2001
From: "Max.Bed4d" <max.bed4d@googlemail.com>
Date: Mon, 15 Jan 2024 13:30:07 +0000
Subject: [PATCH 01/17] CI: Update aio jobs to run on ARC

The jobs now run in containers, so some changes are necessary.
---
 .github/workflows/stackhpc-all-in-one.yml     | 25 +++++++++++++------
 .../workflows/stackhpc-build-kayobe-image.yml |  4 +++
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index f9554c31d..c343a5da3 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -70,17 +70,26 @@ jobs:
   all-in-one:
     name: All in one
     if: inputs.if
-    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    runs-on: arc-skc-aio-runner
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: build-essential git unzip nodejs python3 python3-pip python3-wheel python-is-python3
+
       - uses: actions/checkout@v4
         with:
            submodules: true
 
+      - name: Make sure dockerd is running and test Docker.
+        run: |
+          docker run --rm hello-world
+
       - name: Output image tag
         id: image_tag
         run: |
@@ -209,11 +218,11 @@ jobs:
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
       - name: Pull latest Kayobe image
         run: |
-          sudo docker image pull $KAYOBE_IMAGE
+          docker image pull $KAYOBE_IMAGE
 
       - name: Run growroot
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -223,7 +232,7 @@ jobs:
 
       - name: Host configure
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -233,7 +242,7 @@ jobs:
 
       - name: Service deploy
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -243,7 +252,7 @@ jobs:
 
       - name: Configure aio resources
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -254,7 +263,7 @@ jobs:
       - name: Tempest tests
         run: |
           mkdir -p tempest-artifacts
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
@@ -284,5 +293,5 @@ jobs:
 
       - name: Prune Docker images over 1 week old
         # May fail if another prune is running
-        run: sudo docker image prune --all --force --filter until=168h || true
+        run: docker image prune --all --force --filter until=168h || true
         if: always()
diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml
index 0fbc5c7ce..330de33d3 100644
--- a/.github/workflows/stackhpc-build-kayobe-image.yml
+++ b/.github/workflows/stackhpc-build-kayobe-image.yml
@@ -78,6 +78,8 @@ jobs:
            https_proxy: ${{ inputs.https_proxy }}
            no_proxy: ${{ inputs.no_proxy }}
 
+      # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
+      # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
         uses: docker/build-push-action@v3
         with:
@@ -86,6 +88,8 @@ jobs:
           build-args: |
              http_proxy=${{ inputs.http_proxy }}
              https_proxy=${{ inputs.https_proxy }}
+             KAYOBE_USER_UID=1001
+             KAYOBE_USER_GID=1001
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

From ee19b5b51699165bb920b57195f95641640be42a Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 11:47:57 +0000
Subject: [PATCH 02/17] Support credentials for custom DNF repositories

This allows us to use packages from authenticated repositories such as
those on Ark. This patch depends on
https://review.opendev.org/c/openstack/kayobe/+/908142/
---
 etc/kayobe/dnf.yml      | 34 ++++++++++++++++++++++++++++++++++
 etc/kayobe/stackhpc.yml |  6 ++++++
 2 files changed, 40 insertions(+)

diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index b060c478f..672d14b8b 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -61,6 +61,8 @@ dnf_custom_repos_el8:
     file: epel
     gpgkey: "{{ dnf_epel_8_gpg_key_url }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   epel-modular:
     baseurl: "{{ stackhpc_repo_epel_modular_url }}"
     description: "Extra Packages for Enterprise Linux Modular $releasever - $basearch"
@@ -68,6 +70,8 @@ dnf_custom_repos_el8:
     file: epel-modular
     gpgkey: "{{ dnf_epel_8_gpg_key_url }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   docker:
     baseurl: "{{ stackhpc_repo_docker_url }}"
     description: "Package repository for installing docker"
@@ -75,6 +79,8 @@ dnf_custom_repos_el8:
     file: docker
     gpgkey: "{{ dnf_docker_gpg_key_url }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # CentOS Stream 8 specific repositories
 dnf_custom_repos_centos:
@@ -84,6 +90,8 @@ dnf_custom_repos_centos:
     file: CentOS-Stream-AppStream
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   baseos:
     baseurl: "{{ stackhpc_repo_centos_stream_baseos_url }}"
     description: "CentOS Stream $releasever - BaseOS"
@@ -92,6 +100,8 @@ dnf_custom_repos_centos:
     gpgcheck: yes
     # Exclude buggy iptables: https://bugzilla.redhat.com/show_bug.cgi?id=2236501
     exclude: "iptables-1.8.5* iptables-libs-1.8.5* iptables-ebtables-1.8.5* iptables-services-1.8.5* iptables-arptables-1.8.5* iptables-devel-1.8.5*"
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   extras:
     # This repo is no longer updated: https://www.spinics.net/lists/centos-devel/msg21454.html
     baseurl: "{{ stackhpc_repo_centos_stream_extras_url }}"
@@ -100,12 +110,16 @@ dnf_custom_repos_centos:
     enabled: false
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   extras-common:
     baseurl: "{{ stackhpc_repo_centos_stream_extras_common_url }}"
     description: "CentOS Stream $releasever - Extras common Packages"
     file: CentOS-Stream-Extras-common
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Extras
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Rocky 8 specific repositories
 dnf_custom_repos_rocky:
@@ -115,18 +129,24 @@ dnf_custom_repos_rocky:
     file: "{{ 'Rocky-AppStream' if os_release == '8' else 'rocky' }}"
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   baseos:
     baseurl: "{{ stackhpc_repo_rocky_baseos_url }}"
     description: "Rocky Linux $releasever - BaseOS"
     file: "{{ 'Rocky-BaseOS' if os_release == '8' else 'rocky' }}"
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   extras:
     baseurl: "{{ stackhpc_repo_rocky_extras_url }}"
     description: "Rocky Linux $releasever - Extras"
     file: "{{ 'Rocky-Extras' if os_release == '8' else 'rocky-extras' }}"
     gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Custom repositories shared between all RHEL 9 derivatives.
 dnf_custom_repos_el9:
@@ -137,6 +157,8 @@ dnf_custom_repos_el9:
     file: epel
     gpgkey: "{{ dnf_epel_9_gpg_key_url }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   docker:
     baseurl: "{{ stackhpc_repo_centos_stream_9_docker_url }}"
     description: "Package repository for installing docker"
@@ -144,6 +166,8 @@ dnf_custom_repos_el9:
     file: docker
     gpgkey: "{{ dnf_docker_gpg_key_url }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # ELRepo 9
 dnf_custom_repos_elrepo_9:
@@ -154,6 +178,8 @@ dnf_custom_repos_elrepo_9:
     file: elrepo
     gpgkey: https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Rocky 9 specific repositories
 dnf_custom_repos_rocky_9:
@@ -163,24 +189,32 @@ dnf_custom_repos_rocky_9:
     file: rocky
     gpgkey: "{{ rocky_9_gpg_key }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   baseos:
     baseurl: "{{ stackhpc_repo_rocky_9_baseos_url }}"
     description: "Rocky Linux $releasever - BaseOS"
     file: rocky
     gpgkey: "{{ rocky_9_gpg_key }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   crb:
     baseurl: "{{ stackhpc_repo_rocky_9_crb_url }}"
     description: "Rocky Linux $releasever - CRB"
     file: rocky
     gpgkey: "{{ rocky_9_gpg_key }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
   extras:
     baseurl: "{{ stackhpc_repo_rocky_9_extras_url }}"
     description: "Rocky Linux $releasever - Extras"
     file: rocky-extras
     gpgkey: "{{ rocky_9_gpg_key }}"
     gpgcheck: yes
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Whether to enable EPEL repositories. This affects RedHat-based systems only.
 dnf_enable_epel: "{{ dnf_install_epel | bool }}"
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 3c0e32430..21233ab34 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -2,6 +2,12 @@
 # Host and port of a package repository mirror.
 stackhpc_repo_mirror_url: "{{ pulp_url }}"
 
+# Username of a package repository mirror.
+stackhpc_repo_mirror_username:
+
+# Password of a package repository mirror.
+stackhpc_repo_mirror_password:
+
 # Distribution name. Either 'development' or 'production'.
 stackhpc_repo_distribution: "development"
 

From e9130b9c51161fdadd676932eae5f2c13f5948a8 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 11:52:32 +0000
Subject: [PATCH 03/17] CI: Use Ark package repositories to install packages

Previously we were using Test Pulp on SMS lab, but this is out of
action. Switching to Ark allows CI jobs to run on Leafcloud (or anywhere
with Internet access).
---
 etc/kayobe/environments/ci-aio/stackhpc-ci.yml  | 17 ++++++++++-------
 .../environments/ci-builder/stackhpc-ci.yml     |  5 ++++-
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
index 9740da775..6050bffd2 100644
--- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
@@ -17,7 +17,10 @@ resolv_is_managed: false
 
 # Host and port of a package repository mirror.
 # Build and deploy the development Pulp service repositories.
-stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080"
+# Use Ark's package repositories to install packages.
+stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
 
 # Build and deploy released Pulp repository versions.
 stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
@@ -74,11 +77,11 @@ stackhpc_include_os_minor_version_in_repo_url: true
 stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}"
 
 # Username and password of container registry.
-stackhpc_docker_registry_username: "admin"
+stackhpc_docker_registry_username: "release-train-ci"
 stackhpc_docker_registry_password: !vault |
   $ANSIBLE_VAULT;1.1;AES256
-  38653432663133653430323937626239323836653537633633326265643930653034306532646464
-  3766353637326534643136326533633034666535323364660a633639393433376564363666663233
-  38333133393730633666613965653364316162353337313330346164303631313731646461363461
-  3963323635373866630a633533376339363734626664333765313665623662613764363038383735
-  38646138376438643533376161376634653439386230353365316239613430363338
+  38356134376436656165303634626531653836366233383531343439646433376334396438373735
+  3135643664353934356237376134623235356137383263300a333165386562396134633534376532
+  34386133383366326639353432386235336132663839333337323739633434613934346462363031
+  3265323831663964360a643962346231386462323236373963633066393736323234303833363535
+  3664
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index efe4236e9..946759718 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -42,7 +42,10 @@ resolv_is_managed: false
 
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
-stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080"
+# Use Ark's package repositories to install packages.
+stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
+stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
 
 # Build against released Pulp repository versions.
 stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"

From c22bc405500cfa5e0b81c519dabf99081338f571 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 14:35:55 +0000
Subject: [PATCH 04/17] CI: Switch from sms-lab-release to openstack cloud

This more generic name should be more portable.
---
 .github/workflows/stackhpc-pull-request.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index 2d71792a1..3b4a095f0 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -85,7 +85,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -99,7 +99,7 @@ jobs:
     with:
       kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -115,7 +115,7 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -131,7 +131,7 @@ jobs:
       os_distribution: rocky
       os_release: "8"
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -148,7 +148,7 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -165,7 +165,7 @@ jobs:
       os_release: focal
       ssh_username: ubuntu
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -182,7 +182,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -199,7 +199,7 @@ jobs:
       os_release: jammy
       ssh_username: ubuntu
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -216,7 +216,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovs
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
@@ -233,7 +233,7 @@ jobs:
       os_release: "9"
       ssh_username: cloud-user
       neutron_plugin: ovn
-      OS_CLOUD: sms-lab-release
+      OS_CLOUD: openstack
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

From 7f014ad6b6ccf6fef5bd4c1d464d1c1de8749910 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 14:47:58 +0000
Subject: [PATCH 05/17] CI: Update aio VM flavor, network and subnet for
 Leafcloud

---
 .github/workflows/stackhpc-all-in-one.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index c343a5da3..39744b57d 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -38,15 +38,15 @@ on:
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: general.v1.medium
+        default: en1.medium
       vm_network:
         description: Network for the all-in-one VM
         type: string
-        default: stackhpc-release
+        default: stackhpc-ci
       vm_subnet:
         description: Subnet for the all-in-one VM
         type: string
-        default: stackhpc-release-subnet
+        default: stackhpc-ci
       OS_CLOUD:
         description: Name of cloud in clouds.yaml
         type: string

From bc99a536be757f5755b3db2ceb3ceca6f7f9b338 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 14:51:31 +0000
Subject: [PATCH 06/17] CI: Update GitHub action versions for node20

node16 support is deprecated and will be dropped.
---
 .github/workflows/overcloud-host-image-build.yml     | 12 ++++++------
 .github/workflows/stackhpc-all-in-one.yml            |  4 ++--
 .github/workflows/stackhpc-build-kayobe-image.yml    |  8 ++++----
 .github/workflows/stackhpc-container-image-build.yml |  2 +-
 .github/workflows/stackhpc-pull-request.yml          |  4 ++--
 5 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml
index 00952f2cb..0c2fc3efb 100644
--- a/.github/workflows/overcloud-host-image-build.yml
+++ b/.github/workflows/overcloud-host-image-build.yml
@@ -362,7 +362,7 @@ jobs:
         if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload updated images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Updated images list
           path: /tmp/updated_images.txt
@@ -374,7 +374,7 @@ jobs:
             steps.build_ubuntu_jammy.outcome == 'success'
 
       - name: Upload CentOS build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: CentOS build logs
           path: |
@@ -384,7 +384,7 @@ jobs:
         if: steps.build_centos_stream_8.outcome == 'failure'
 
       - name: Upload Rocky 8 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Rocky 8 build logs
           path: |
@@ -394,7 +394,7 @@ jobs:
         if: steps.build_rocky_8.outcome == 'failure'
 
       - name: Upload Rocky 9 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Rocky 9 build logs
           path: |
@@ -404,7 +404,7 @@ jobs:
         if: steps.build_rocky_9.outcome == 'failure'
 
       - name: Upload Ubuntu Focal 20.04 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Ubuntu Focal 20.04 build logs
           path: |
@@ -414,7 +414,7 @@ jobs:
         if: steps.build_ubuntu_focal.outcome == 'failure'
 
       - name: Upload Ubuntu Jammy 22.04 build logs if build failed
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Ubuntu Jammy 22.04 build logs
           path: |
diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 39744b57d..3a6f1dc84 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -106,7 +106,7 @@ jobs:
           fi
 
       - name: Install terraform
-        uses: hashicorp/setup-terraform@v2
+        uses: hashicorp/setup-terraform@v3
 
       - name: Initialise terraform
         run: terraform init
@@ -273,7 +273,7 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
       - name: Upload test result artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}
           path: tempest-artifacts/*
diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml
index 330de33d3..ace9a544c 100644
--- a/.github/workflows/stackhpc-build-kayobe-image.yml
+++ b/.github/workflows/stackhpc-build-kayobe-image.yml
@@ -52,7 +52,7 @@ jobs:
            submodules: true
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -60,12 +60,12 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           driver-opts: |
             image=moby/buildkit:master
@@ -81,7 +81,7 @@ jobs:
       # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults
       # so that docker can run as a privileged user within the Kayobe image.
       - name: Build and push Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: ./.automation/docker/kayobe/Dockerfile
           context: .
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index d899f20b3..5309b21d5 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -209,7 +209,7 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Upload container images artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.distro }} container images
           path: ${{ matrix.distro }}-container-images
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index 3b4a095f0..987e15356 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Check changed files
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           # Filters are defined in this file.
@@ -51,7 +51,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python ${{ matrix.python-version }} 🐍
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Tox 📦

From 1cd35093a9666166152fed0c43a5ce5558e083d3 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 8 Feb 2024 12:52:52 +0000
Subject: [PATCH 07/17] Revert hashicorp/setup-terraform action to v2 due to
 failure

---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 3a6f1dc84..8c5b673ff 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -106,7 +106,7 @@ jobs:
           fi
 
       - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
+        uses: hashicorp/setup-terraform@v2
 
       - name: Initialise terraform
         run: terraform init

From cc5199745180bf07b45c5177001c50b419b6f7c7 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 7 Feb 2024 15:32:47 +0000
Subject: [PATCH 08/17] CI: Revert to ens3 interface for Leafcloud

Leafcloud is not using the q35 machine type yet.
---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 8c5b673ff..922bf5d54 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -34,7 +34,7 @@ on:
       vm_interface:
         description: Default network interface name
         type: string
-        default: enp3s0
+        default: ens3
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string

From 3abdf8c98b0c46c2738a961377e7ad215dbf36c1 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 8 Feb 2024 11:59:15 +0000
Subject: [PATCH 09/17] Add Apt auth configuration

This allows us to use packages from authenticated repositories such as
those on Ark. This patch depends on
https://review.opendev.org/c/openstack/kayobe/+/908413
---
 etc/kayobe/apt.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml
index 862b21956..1d043d111 100644
--- a/etc/kayobe/apt.yml
+++ b/etc/kayobe/apt.yml
@@ -75,6 +75,22 @@ apt_repositories: "{{ stackhpc_apt_repositories if 'overcloud' in group_names el
 # repository mirrors are deployed.
 apt_disable_sources_list: "{{ 'overcloud' in group_names }}"
 
+# Apt auth configuration for accessing the package repository mirror.
+stackhpc_apt_auth:
+  - machine: "{{ stackhpc_repo_mirror_url }}"
+    login: "{{ stackhpc_repo_mirror_username }}"
+    password: "{{ stackhpc_repo_mirror_password }}"
+    filename: "kayobe.conf"
+
+# List of Apt auth configurations. Each item is a dict with the following keys:
+# * machine: 'machine' entry in the auth file
+# * login: 'login' entry in the auth file
+# * password: 'password' entry in the auth file
+# * filename: Name of a file in which to store the auth configuration. The
+#   extension should be '.conf'.
+# Default is an empty list.
+apt_auth: "{{ stackhpc_apt_auth if 'overcloud' in group_names and stackhpc_repo_mirror_username is truthy else [] }}"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

From af7316123cdb27f51dc53b7a374510154d02b6a5 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 8 Feb 2024 12:22:52 +0000
Subject: [PATCH 10/17] CI: Increase flavor to en1.large - some jobs were
 failing Tempest jobs

---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 922bf5d54..75aff0f82 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -38,7 +38,7 @@ on:
       vm_flavor:
         description: Flavor for the all-in-one VM
         type: string
-        default: en1.medium
+        default: en1.large
       vm_network:
         description: Network for the all-in-one VM
         type: string

From dbf3508b8edcb9883780229093699a0428f78d03 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 23 Jan 2024 10:03:44 +0000
Subject: [PATCH 11/17] CI: Run container image build job on ARC cluster

The Actions Runner Controller (ARC) cluster runs GitHub Actions jobs on
a Kubernetes cluster. This allows us to dynamically provision resources
for our CI workflows.

This change switches the container image build jobs to use the ARC
cluster. The job now runs in a container, with access to a Docker in
Docker container for building images. This allows us to skip the host
configuration part of the workflow.

See https://github.com/stackhpc/ARC-Installer/
---
 .github/workflows/stackhpc-all-in-one.yml     |  5 --
 .../stackhpc-container-image-build.yml        | 66 +++++++------------
 2 files changed, 22 insertions(+), 49 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 75aff0f82..ba6fa26cb 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -290,8 +290,3 @@ jobs:
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: always()
-
-      - name: Prune Docker images over 1 week old
-        # May fail if another prune is running
-        run: docker image prune --all --force --filter until=168h || true
-        if: always()
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 5309b21d5..296637775 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -45,7 +45,7 @@ jobs:
   generate-tag:
     name: Generate container image tag
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: ubuntu-latest
     permissions: {}
     outputs:
       kolla_tag: ${{ steps.kolla_tag.outputs.kolla_tag }}
@@ -97,7 +97,7 @@ jobs:
   container-image-build:
     name: Build Kolla container images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: arc-skc-container-image-builder-runner
     timeout-minutes: 720
     permissions: {}
     strategy:
@@ -106,7 +106,13 @@ jobs:
     needs:
       - generate-tag
     steps:
-      - uses: actions/checkout@v4
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           path: src/kayobe-config
 
@@ -117,33 +123,9 @@ jobs:
           ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
           path: src/kayobe
 
-      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
-      - name: Install dependencies
+      - name: Make sure dockerd is running and test Docker
         run: |
-          sudo dnf -y install python3-virtualenv
-
-      - name: Setup networking
-        run: |
-          if ! ip l show breth1 >/dev/null 2>&1; then
-              sudo ip l add breth1 type bridge
-          fi
-          sudo ip l set breth1 up
-          if ! ip a show breth1 | grep 192.168.33.3/24; then
-              sudo ip a add 192.168.33.3/24 dev breth1
-          fi
-          if ! ip l show dummy1 >/dev/null 2>&1; then
-              sudo ip l add dummy1 type dummy
-          fi
-          sudo ip l set dummy1 up
-          sudo ip l set dummy1 master breth1
-
-      # FIXME: Without this workaround we see the following issue after the runner is power cycled:
-      # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] ***
-      #   Unable to start service network: Job for network.service failed because the control process exited with error code.
-      #   See \"systemctl status network.service\" and \"journalctl -xe\" for details.
-      - name: Kill dhclient (workaround)
-        run: |
-          (sudo killall dhclient || true) && sudo systemctl restart network
+          docker run --rm hello-world
 
       - name: Install Kayobe
         run: |
@@ -154,23 +136,19 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      - name: Bootstrap the control host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe control host bootstrap
-
-      - name: Configure the seed host
+      # Required for Docker registry login. Normally installed during host configure.
+      - name: Install Docker Python SDK
         run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+          pip install --user docker
 
-      - name: Prune local Kolla container images over 1 week old
+      - name: Configure localhost as a seed
         run: |
-          sudo docker image prune --all --force --filter until=168h --filter="label=kolla_version"
+          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
+          # A 'seed' host used for building images.
+          # Use localhost for container image builds.
+          [seed]
+          localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
+          EOF
 
       - name: Build and push kolla overcloud images
         run: |
@@ -203,7 +181,7 @@ jobs:
 
       - name: Get built container images
         run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
+          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi

From 05e1995f6553e617f6bbc0712e9e1e7be18639c9 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 8 Feb 2024 11:56:54 +0000
Subject: [PATCH 12/17] Add package repository credentials to container image
 build

This is necessary to access package repositories hosted on Ark.
---
 etc/kayobe/kolla.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 082e9243b..45c9b573c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -317,12 +317,22 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
+    {% if stackhpc_repo_mirror_username is truthy %}
+               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
+               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
+    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
+        rm -f /etc/apt/auth.conf && \
+    {% if stackhpc_repo_mirror_username is truthy %}
+        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
+        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
+        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
+    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -340,6 +350,10 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
+    {% if stackhpc_repo_mirror_username is truthy %}
+               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
+               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
+    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
     {% endif %}
@@ -350,6 +364,12 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
+        rm -f /etc/apt/auth.conf && \
+    {% if stackhpc_repo_mirror_username is truthy %}
+        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
+        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
+        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
+    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}

From f8947a99318d73827de2653d8027f69a6df31f3e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Thu, 8 Feb 2024 13:04:10 +0000
Subject: [PATCH 13/17] Don't verify Apt repo CA initially when using HTTPS in
 container build

The Ubuntu base image doesn't contain the ca-certificates package, so we
can't verify an HTTPS package mirror to download the ca-certificates
package. Using the upstream repos may result in a version conflict when
we switch to another mirror.
---
 etc/kayobe/kolla.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 45c9b573c..15407f00c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -328,6 +328,10 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
+    {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %}
+    {# We lack the ca-certificates package at this stage, so don't verify the CA #}
+        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
+    {% endif %}
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
@@ -365,6 +369,7 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
+        rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \

From e1e9f5dc4030eed7dc8a6f9234b1b201404e79e3 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 09:39:38 +0000
Subject: [PATCH 14/17] CI: Fix up check-tags job for ARC

---
 .github/workflows/stackhpc-check-tags.yml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml
index 1fdb01307..0ef1526ae 100644
--- a/.github/workflows/stackhpc-check-tags.yml
+++ b/.github/workflows/stackhpc-check-tags.yml
@@ -20,29 +20,33 @@ jobs:
   check-tags:
     name: Check container image tags
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-aio]
+    runs-on: arc-skc-aio-runner
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Install package dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y git unzip nodejs
+
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           submodules: true
 
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
       - name: Pull latest Kayobe image
         run: |
-          sudo docker image pull $KAYOBE_IMAGE
+          docker image pull $KAYOBE_IMAGE
 
       - name: Check container image tags
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
             /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
             '$KAYOBE_CONFIG_PATH/ansible/check-tags.yml'
-        #env:
-          #KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

From 8d9e620717217745b41726cd7f7100bc8d6bd53c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 09:46:17 +0000
Subject: [PATCH 15/17] CI: Trim down package dependencies

---
 .github/workflows/stackhpc-all-in-one.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index c57ad9917..ed1114191 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -88,7 +88,7 @@ jobs:
       - name: Install Package
         uses: ConorMacBride/install-package@main
         with:
-          apt: build-essential git unzip nodejs python3 python3-pip python3-wheel python-is-python3
+          apt: git unzip nodejs
 
       # If testing upgrade, checkout previous release, otherwise checkout current branch
       - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config

From 6329f1c83b18f1db7a0ceb875126249c62d24a8e Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 11:26:48 +0000
Subject: [PATCH 16/17] CI: Add retries to TF apply in aio job

We sometimes hit temporary failures during VM creation. Adding a retry
may help to keep things running.
---
 .github/workflows/stackhpc-all-in-one.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index ed1114191..59077edc1 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -177,7 +177,17 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
       - name: Terraform Apply
-        run: terraform apply -auto-approve
+        run: |
+          for attempt in $(seq 5); do
+              if terraform apply -auto-approve; then
+                  echo "Created infrastructure on attempt $attempt"
+                  break
+              fi
+              echo "Failed to create infrastructure on attempt $attempt"
+              sleep 10
+              terraform destroy -auto-approve
+              sleep 60
+          done
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
           OS_CLOUD: ${{ inputs.OS_CLOUD }}

From eafa278220d62eb7e933f07b8938e0b20201f3a7 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 9 Feb 2024 11:50:26 +0000
Subject: [PATCH 17/17] CI: Fix up aio upgrade job for ARC runner

This job is not present in yoga, so was not tested there.
---
 .github/workflows/stackhpc-all-in-one.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 59077edc1..6831ad826 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -251,7 +251,7 @@ jobs:
       # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
       - name: Pull previous Kayobe image
         run: |
-          sudo docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
+          docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
         if: inputs.upgrade
 
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
@@ -281,7 +281,7 @@ jobs:
 
       - name: Service deploy
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             ${{ steps.kayobe_image.outputs.kayobe_image }} \
@@ -291,7 +291,7 @@ jobs:
 
       - name: Configure aio resources
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             ${{ steps.kayobe_image.outputs.kayobe_image }} \
@@ -319,7 +319,7 @@ jobs:
       # Now begin upgrade
       - name: Host upgrade
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
@@ -330,7 +330,7 @@ jobs:
 
       - name: Host configure
         run: |
-          sudo -E docker run -t --rm \
+          docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \