From bcae6dc7b8850869c685429cc388c063661a8bdf Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Mon, 31 Jan 2022 12:49:43 +0100
Subject: [PATCH 01/20] Sync kayobe-config with kayobe changes

Change-Id: I39113b819473e3b4a3819e2e03a69e59f3c13c46
---
 etc/kayobe/apt.yml     |  6 ++++++
 etc/kayobe/bifrost.yml |  9 ++++++++-
 etc/kayobe/globals.yml | 11 +++++++++++
 etc/kayobe/ipa.yml     |  4 ++--
 etc/kayobe/kolla.yml   |  1 -
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml
index 552a116cf8..5f278e3222 100644
--- a/etc/kayobe/apt.yml
+++ b/etc/kayobe/apt.yml
@@ -5,6 +5,12 @@
 # Apt cache TTL in seconds. Default is 3600.
 #apt_cache_valid_time:
 
+# Apt proxy URL for HTTP. Default is empty (no proxy).
+#apt_proxy_http:
+
+# Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}.
+#apt_proxy_https:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml
index 91c8bcc567..1921952054 100644
--- a/etc/kayobe/bifrost.yml
+++ b/etc/kayobe/bifrost.yml
@@ -11,6 +11,10 @@
 # {{ openstack_branch }}.
 #kolla_bifrost_source_version:
 
+# Whether Bifrost uses firewalld. Default value is false to avoid conflicting
+# with iptables rules configured on the seed host by Kayobe.
+#kolla_bifrost_use_firewalld:
+
 # Firewalld zone used by Bifrost. Default is "trusted", to avoid blocking other
 # services running on the seed host.
 #kolla_bifrost_firewalld_internal_zone:
@@ -40,7 +44,7 @@
 #kolla_bifrost_dib_init_element:
 
 # DIB default environment variables. Default is {"DIB_CLOUD_INIT_DATASOURCES":
-# "ConfigDrive", "DIB_DISABLE_KERNEL_CLEANUP": 1}.
+# "ConfigDrive"}.
 #kolla_bifrost_dib_env_vars_default:
 
 # DIB additional environment variables. Default is none.
@@ -74,6 +78,9 @@
 #kolla_bifrost_inspector_port_addition:
 
 # List of extra kernel parameters for the inspector default PXE configuration.
+# Default is {{ inspector_extra_kernel_options }}, defined in inspector.yml.
+# When customising this variable, the default extra kernel parameters should be
+# kept to retain full node inspection capabilities.
 #kolla_bifrost_inspector_extra_kernel_options:
 
 # List of introspection rules for Bifrost's Ironic Inspector service.
diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml
index 9efc114f60..a4150d8eca 100644
--- a/etc/kayobe/globals.yml
+++ b/etc/kayobe/globals.yml
@@ -53,6 +53,17 @@
 # "focal" when os_distribution is "ubuntu".
 #os_release:
 
+###############################################################################
+# Ansible configuration.
+
+# Filter to apply to the setup module when gathering facts. Default is to not
+# specify a filter.
+#kayobe_ansible_setup_filter:
+
+# Gather subset to apply to the setup module when gathering facts. Default is
+# to not specify a gather subset.
+#kayobe_ansible_setup_gather_subset:
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml
index 49236d13aa..519ca2a59d 100644
--- a/etc/kayobe/ipa.yml
+++ b/etc/kayobe/ipa.yml
@@ -16,8 +16,8 @@
 # URL of IPA builder source repository.
 #ipa_builder_source_url:
 
-# Version of IPA builder source repository. Default is master.
-#ipa_build_source_version:
+# Version of IPA builder source repository. Default is {{ openstack_branch }}.
+#ipa_builder_source_version:
 
 # List of default Diskimage Builder (DIB) elements to use when building IPA
 # images. Default is ["centos", "enable-serial-console",
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index b43a3b907f..0843df106c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -265,7 +265,6 @@
 #kolla_enable_cinder_backend_lvm:
 #kolla_enable_cinder_backend_nfs:
 #kolla_enable_cinder_backend_quobyte:
-#kolla_enable_cinder_backend_zfssa_iscsi:
 #kolla_enable_cinder_backup:
 #kolla_enable_cinder_horizon_policy_file:
 #kolla_enable_cloudkitty:

From 86c9655db27fbafbd257797906897e46d483eae8 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Wed, 18 May 2022 10:13:39 +0200
Subject: [PATCH 02/20] [CI] Move queue setting to project level

Per [1].

[1] http://lists.zuul-ci.org/pipermail/zuul-discuss/2022-May/001801.html

Change-Id: I034d7dc4ea40654221d8a748f0fbe836a2e729e5
(cherry picked from commit b6e75b5cd6011886f87ba81c29996b1adb1f2fd9)
---
 zuul.d/project.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 4cee999095..2551dd112e 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -1,10 +1,10 @@
 ---
 - project:
+    queue: kayobe
     check:
       jobs:
         - openstack-tox-pep8
 
     gate:
-      queue: kayobe
       jobs:
         - openstack-tox-pep8

From 1880a732e3abf1475abef6d5281338e042d77dfb Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Thu, 22 Sep 2022 11:15:53 +0100
Subject: [PATCH 03/20] playbook for installing and running cardiff

---
 etc/kayobe/ansible/cardiff-run.yml | 44 ++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 etc/kayobe/ansible/cardiff-run.yml

diff --git a/etc/kayobe/ansible/cardiff-run.yml b/etc/kayobe/ansible/cardiff-run.yml
new file mode 100644
index 0000000000..0d2676e30b
--- /dev/null
+++ b/etc/kayobe/ansible/cardiff-run.yml
@@ -0,0 +1,44 @@
+---
+- name: Cardiff run
+  hosts: localhost
+  gather_facts: no
+  tags:
+    - cardiff
+  vars:
+    venv: "~/venvs/cardiff-venv"
+    input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
+    output_dir: "{{ lookup('env', 'PWD') }}/review"
+  tasks:
+    - name: Install dependencies
+      pip:
+        virtualenv: "{{ venv }}"
+        name:
+          - git+https://github.com/stackhpc/cardiff
+          - pyvis
+          - pyvis.network
+        state: latest
+
+    - name: Create data directory
+      file:
+        path: '{{ output_dir }}/data'
+        state: directory
+
+    - name: Extract data
+      shell:
+        cmd: >
+          {{ venv }}/bin/m2-extract {{ input_dir }}/*.json --output_dir {{ output_dir }}/data
+
+    - name: Create review directory
+      file:
+        path: '{{ output_dir }}/results'
+        state: directory
+
+    - name: Process data
+      shell:
+        cmd: >
+          {{ venv }}/bin/cardiff
+          -I ipmi
+          -p '{{ output_dir }}/data/extra-hardware/*.eval'
+          -o '{{ output_dir }}/results'
+          --visualise
+        #  --visualise_dataless
\ No newline at end of file

From a1bccbc8221d881ddbc4919c6751208767c5a328 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Thu, 22 Sep 2022 14:37:05 +0100
Subject: [PATCH 04/20] Add regex pattern, add line to eof

---
 etc/kayobe/ansible/cardiff-run.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/cardiff-run.yml b/etc/kayobe/ansible/cardiff-run.yml
index 0d2676e30b..a2090682d6 100644
--- a/etc/kayobe/ansible/cardiff-run.yml
+++ b/etc/kayobe/ansible/cardiff-run.yml
@@ -8,12 +8,14 @@
     venv: "~/venvs/cardiff-venv"
     input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
     output_dir: "{{ lookup('env', 'PWD') }}/review"
+    cardiff_pattern: ".*.eval" # Uses regex
   tasks:
     - name: Install dependencies
       pip:
         virtualenv: "{{ venv }}"
         name:
           - git+https://github.com/stackhpc/cardiff
+          #- /home/matt/Documents/cardiff
           - pyvis
           - pyvis.network
         state: latest
@@ -38,7 +40,7 @@
         cmd: >
           {{ venv }}/bin/cardiff
           -I ipmi
-          -p '{{ output_dir }}/data/extra-hardware/*.eval'
+          -p '{{ output_dir }}/data/extra-hardware/{{ cardiff_pattern }}'
           -o '{{ output_dir }}/results'
           --visualise
-        #  --visualise_dataless
\ No newline at end of file
+        #  --visualise_dataless

From 3df7cb4295246dd559f1a7cfe45a253fbded1e7c Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 7 Oct 2022 09:32:07 +0100
Subject: [PATCH 05/20] Review changes

---
 etc/kayobe/ansible/cardiff-run.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/cardiff-run.yml b/etc/kayobe/ansible/cardiff-run.yml
index a2090682d6..e51a7858d8 100644
--- a/etc/kayobe/ansible/cardiff-run.yml
+++ b/etc/kayobe/ansible/cardiff-run.yml
@@ -5,7 +5,7 @@
   tags:
     - cardiff
   vars:
-    venv: "~/venvs/cardiff-venv"
+    venv: "~/venvs/cardiff-review"
     input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
     output_dir: "{{ lookup('env', 'PWD') }}/review"
     cardiff_pattern: ".*.eval" # Uses regex
@@ -15,7 +15,6 @@
         virtualenv: "{{ venv }}"
         name:
           - git+https://github.com/stackhpc/cardiff
-          #- /home/matt/Documents/cardiff
           - pyvis
           - pyvis.network
         state: latest
@@ -43,4 +42,3 @@
           -p '{{ output_dir }}/data/extra-hardware/{{ cardiff_pattern }}'
           -o '{{ output_dir }}/results'
           --visualise
-        #  --visualise_dataless

From 9a48295a47fb1e67414ff3ac1d8c01cd61819438 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Fri, 14 Oct 2022 11:10:41 +0100
Subject: [PATCH 06/20] packages moved to requirements file

---
 etc/kayobe/ansible/cardiff-run.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/etc/kayobe/ansible/cardiff-run.yml b/etc/kayobe/ansible/cardiff-run.yml
index e51a7858d8..0b796a46e0 100644
--- a/etc/kayobe/ansible/cardiff-run.yml
+++ b/etc/kayobe/ansible/cardiff-run.yml
@@ -15,8 +15,6 @@
         virtualenv: "{{ venv }}"
         name:
           - git+https://github.com/stackhpc/cardiff
-          - pyvis
-          - pyvis.network
         state: latest
 
     - name: Create data directory

From f9ec2b1597de7a6ff6fac6a68e2d789b5a5e46b5 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Tue, 18 Oct 2022 15:03:10 +0100
Subject: [PATCH 07/20] Change name from cardiff to ADVise

---
 .../ansible/{cardiff-run.yml => advise-run.yml}    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
 rename etc/kayobe/ansible/{cardiff-run.yml => advise-run.yml} (74%)

diff --git a/etc/kayobe/ansible/cardiff-run.yml b/etc/kayobe/ansible/advise-run.yml
similarity index 74%
rename from etc/kayobe/ansible/cardiff-run.yml
rename to etc/kayobe/ansible/advise-run.yml
index 0b796a46e0..c3b6f2ffdf 100644
--- a/etc/kayobe/ansible/cardiff-run.yml
+++ b/etc/kayobe/ansible/advise-run.yml
@@ -1,20 +1,20 @@
 ---
-- name: Cardiff run
+- name: ADVise run
   hosts: localhost
   gather_facts: no
   tags:
-    - cardiff
+    - advise
   vars:
-    venv: "~/venvs/cardiff-review"
+    venv: "~/venvs/advise-review"
     input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
     output_dir: "{{ lookup('env', 'PWD') }}/review"
-    cardiff_pattern: ".*.eval" # Uses regex
+    advise_pattern: ".*.eval" # Uses regex
   tasks:
     - name: Install dependencies
       pip:
         virtualenv: "{{ venv }}"
         name:
-          - git+https://github.com/stackhpc/cardiff
+          - git+https://github.com/stackhpc/ADVise
         state: latest
 
     - name: Create data directory
@@ -35,8 +35,8 @@
     - name: Process data
       shell:
         cmd: >
-          {{ venv }}/bin/cardiff
+          {{ venv }}/bin/advise
           -I ipmi
-          -p '{{ output_dir }}/data/extra-hardware/{{ cardiff_pattern }}'
+          -p '{{ output_dir }}/data/extra-hardware/{{ advise_pattern }}'
           -o '{{ output_dir }}/results'
           --visualise

From 022e0bd4e2ea474b6cdab14e094e9cce5607aac1 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Mon, 24 Oct 2022 09:16:16 +0100
Subject: [PATCH 08/20] Separate processing from visualisation

---
 etc/kayobe/ansible/advise-run.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/advise-run.yml b/etc/kayobe/ansible/advise-run.yml
index c3b6f2ffdf..09440aeb95 100644
--- a/etc/kayobe/ansible/advise-run.yml
+++ b/etc/kayobe/ansible/advise-run.yml
@@ -35,8 +35,12 @@
     - name: Process data
       shell:
         cmd: >
-          {{ venv }}/bin/advise
+          {{ venv }}/bin/advise-process
           -I ipmi
           -p '{{ output_dir }}/data/extra-hardware/{{ advise_pattern }}'
-          -o '{{ output_dir }}/results'
-          --visualise
+          -o '{{ output_dir }}'
+
+    - name: Visualise data
+      command: >
+        {{ venv }}/bin/advise-visualise
+        --output_dir '{{ output_dir }}'

From 7cd8ea0b10843d1a4188187c8592ff2f60b90c55 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 23 Nov 2022 08:42:03 +0000
Subject: [PATCH 09/20] Upgrade Pulp container to 3.21

---
 etc/kayobe/seed.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml
index ce16140fa8..78cb92e79d 100644
--- a/etc/kayobe/seed.yml
+++ b/etc/kayobe/seed.yml
@@ -106,7 +106,7 @@ seed_pulp_container:
     image: pulp/pulp
     pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml"
     post: "{{ kayobe_config_path }}/containers/pulp/post.yml"
-    tag: "3.16"
+    tag: "3.21"
     network_mode: host
     volumes:
       - /opt/kayobe/containers/pulp:/etc/pulp

From 74c1e58bc585429aeda91d3562625c8acef07d5f Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <identity@jackhodgkiss.uk>
Date: Mon, 28 Nov 2022 10:53:32 +0000
Subject: [PATCH 10/20] feat: bump stackhp.pulp from `0.3.0` ~> `0.4.0` (#241)

* feat: bump stackhp.pulp from `0.3.0` ~> `0.4.0`

* fix: remove VXLAN role and set correct collection
---
 etc/kayobe/ansible/requirements.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index d8c1ad3f25..8293d49dc4 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -1,6 +1,6 @@
 ---
 collections:
   - name: stackhpc.pulp
-    version: 0.3.0
+    version: 0.4.0
   - name: pulp.squeezer
     version: 0.0.11

From 15024b27c1771a52ad080f2b6f02f67cc525baef Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 29 Nov 2022 19:59:30 +0000
Subject: [PATCH 11/20] CI: symlink to prometheus & grafana config in ci-aio &
 ci-multinode

Symlinks created via:

  ln -s ../../../../kolla/config/grafana/ etc/kayobe/environments/ci-aio/kolla/config/
  ln -s ../../../../kolla/config/prometheus/ etc/kayobe/environments/ci-aio/kolla/config/

  ln -s ../../../../kolla/config/grafana/ etc/kayobe/environments/ci-multinode/kolla/config/
  ln -s ../../../../kolla/config/prometheus/ etc/kayobe/environments/ci-multinode/kolla/config/
---
 etc/kayobe/environments/ci-aio/kolla/config/grafana          | 1 +
 etc/kayobe/environments/ci-aio/kolla/config/prometheus       | 1 +
 etc/kayobe/environments/ci-multinode/kolla/config/grafana    | 1 +
 etc/kayobe/environments/ci-multinode/kolla/config/prometheus | 1 +
 4 files changed, 4 insertions(+)
 create mode 120000 etc/kayobe/environments/ci-aio/kolla/config/grafana
 create mode 120000 etc/kayobe/environments/ci-aio/kolla/config/prometheus
 create mode 120000 etc/kayobe/environments/ci-multinode/kolla/config/grafana
 create mode 120000 etc/kayobe/environments/ci-multinode/kolla/config/prometheus

diff --git a/etc/kayobe/environments/ci-aio/kolla/config/grafana b/etc/kayobe/environments/ci-aio/kolla/config/grafana
new file mode 120000
index 0000000000..0e711c2ae9
--- /dev/null
+++ b/etc/kayobe/environments/ci-aio/kolla/config/grafana
@@ -0,0 +1 @@
+../../../../kolla/config/grafana/
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-aio/kolla/config/prometheus b/etc/kayobe/environments/ci-aio/kolla/config/prometheus
new file mode 120000
index 0000000000..9a40a2c64c
--- /dev/null
+++ b/etc/kayobe/environments/ci-aio/kolla/config/prometheus
@@ -0,0 +1 @@
+../../../../kolla/config/prometheus/
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/grafana b/etc/kayobe/environments/ci-multinode/kolla/config/grafana
new file mode 120000
index 0000000000..0e711c2ae9
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/kolla/config/grafana
@@ -0,0 +1 @@
+../../../../kolla/config/grafana/
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/prometheus b/etc/kayobe/environments/ci-multinode/kolla/config/prometheus
new file mode 120000
index 0000000000..9a40a2c64c
--- /dev/null
+++ b/etc/kayobe/environments/ci-multinode/kolla/config/prometheus
@@ -0,0 +1 @@
+../../../../kolla/config/prometheus/
\ No newline at end of file

From e264f928a7105eff5ee4140b7ee02ecd574b271f Mon Sep 17 00:00:00 2001
From: Will Szumski <williamszumski@gmail.com>
Date: Fri, 2 Dec 2022 18:59:13 +0000
Subject: [PATCH 12/20] Fixes for Rocky 8 pulp snapshots

---
 etc/kayobe/pulp.yml     | 2 +-
 etc/kayobe/stackhpc.yml | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 5aca09ca30..cdc62c85f9 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -253,7 +253,7 @@ stackhpc_pulp_repository_rpm_repos:
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - PowerTools
-    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_6_powertools_version }}"
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}"
     client_cert: "{{ stackhpc_release_pulp_client_cert }}"
     client_key: "{{ stackhpc_release_pulp_client_key }}"
     policy: on_demand
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 03aa3071a2..f1910e1c19 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -96,15 +96,15 @@ stackhpc_repo_treasuredata_4_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/t
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 BaseOS
-stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}"
+stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}/"
 stackhpc_repo_rocky_baseos_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 AppStream
-stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}"
+stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}/"
 stackhpc_repo_rocky_appstream_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 extras
-stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}"
+stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}/"
 stackhpc_repo_rocky_extras_version: "{{ stackhpc_repo_distribution }}"
 
 ###############################################################################

From d4b449cd30235c1bded079480c1eb1a93356a632 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Mon, 5 Dec 2022 13:25:29 +0100
Subject: [PATCH 13/20] Replace hardcoded threshold by temp_max value

The temp_max value is dynamically gathered from the device [1]. With
Xeon CPUs (coretemp driver), it is often 90C, but sometimes lower.

This can help reduce the frequency of alerts with busy hypervisors.

[1] https://docs.kernel.org/hwmon/coretemp.html
---
 etc/kayobe/kolla/config/prometheus/system.rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules
index fe3a2b9acb..ffc7d25a3c 100644
--- a/etc/kayobe/kolla/config/prometheus/system.rules
+++ b/etc/kayobe/kolla/config/prometheus/system.rules
@@ -34,7 +34,7 @@ groups:
       description: "OOM kill detected"
 
   - alert: Overheating
-    expr: node_hwmon_temp_celsius >= 85
+    expr: node_hwmon_temp_celsius >= node_hwmon_temp_max_celsius
     for: 1m
     labels:
       severity: warning

From e3f83f5111f7565342de31276a68c106fa20af08 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 5 Dec 2022 17:08:35 +0000
Subject: [PATCH 14/20] Adding support for pulp RBAC

---
 README.rst                                    | 32 +++++++--
 .../certs/ark.stackhpc.com/client-cert.pem    |  0
 .../certs/ark.stackhpc.com/client-key.pem     |  0
 etc/kayobe/ansible/requirements.yml           |  4 +-
 etc/kayobe/pulp.yml                           | 67 +++++++++----------
 5 files changed, 58 insertions(+), 45 deletions(-)
 delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem
 delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem

diff --git a/README.rst b/README.rst
index 304a8edbc7..6f62d93d95 100644
--- a/README.rst
+++ b/README.rst
@@ -13,7 +13,7 @@ StackHPC provides packages and container images for OpenStack via `Ark
 
 Deployments should use a local `Pulp <https://pulpproject.org/>`__ repository
 server to synchronise content from Ark and serve it locally. Access to the
-repositories on Ark is controlled via X.509 certificates issued by StackHPC.
+repositories on Ark is controlled via user accounts issued by StackHPC.
 
 This configuration is a base, and should be merged with any existing Kayobe
 configuration. It currently provides the following:
@@ -112,11 +112,6 @@ password:
    stackhpc_release_pulp_username: <username>
    stackhpc_release_pulp_password: <password>
 
-The client certificate and private key issued by StackHPC should be stored in
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively,
-with the private key encrypted via Ansible Vault.
-
 The distribution name for the environment should be configured as either
 ``development`` or ``production`` via ``stackhpc_repo_distribution`` in
 ``etc/kayobe/stackhpc.yml``.
@@ -216,6 +211,31 @@ with the push repository using the pulp CLI:
     Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/
     .Done.
 
+HTTP Error 404: Not Found 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your login credentials are incorrect, or lack the required permissions, 
+you will see a 404 error during ``pulp-repo-sync.yml``:
+
+.. code-block:: console
+    TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] ****************************************************************************************************************************************
+    An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435'))
+    failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false 
+      ansible_loop_var: item
+      item:
+        name: centos-stream-8-baseos-development
+        policy: on_demand
+        proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287
+        remote_password: <password>
+        remote_username: <username>
+        required: true
+        state: present
+        sync_policy: mirror_complete
+        url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435
+      msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) '''
+The issue can be rectified by updating the ``stackhpc_release_pulp_username``
+and ``stackhpc_release_pulp_password`` variables
+
 Environments
 ============
 
diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 8293d49dc4..0ced4d5277 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -1,6 +1,4 @@
 ---
 collections:
   - name: stackhpc.pulp
-    version: 0.4.0
-  - name: pulp.squeezer
-    version: 0.0.11
+    version: 0.4.1
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index cdc62c85f9..d6017895ef 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -19,15 +19,10 @@ pulp_proxy_url: "{{ omit }}"
 # Base URL of the StackHPC Pulp service.
 stackhpc_release_pulp_url: "https://ark.stackhpc.com"
 
-# Credentials used to access the StackHPC Ark container image registry.
+# Credentials used to access the StackHPC Ark pulp server.
 stackhpc_release_pulp_username:
 stackhpc_release_pulp_password:
 
-# Client certificates used to access StackHPC Ark repositories.
-# They are trusted by the 'release' cert guard's CA.
-stackhpc_release_pulp_client_cert: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-cert.pem') | trim }}"
-stackhpc_release_pulp_client_key: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-key.pem') | trim }}"
-
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
@@ -44,8 +39,8 @@ stackhpc_pulp_repository_deb_repos:
   # Base Ubuntu Focal repositories
   - name: Ubuntu focal
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal/{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -57,8 +52,8 @@ stackhpc_pulp_repository_deb_repos:
 
   - name: Ubuntu focal security
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal-security/{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -71,8 +66,8 @@ stackhpc_pulp_repository_deb_repos:
   # Ubuntu Cloud Archive (UCA) repositories
   - name: Ubuntu Cloud Archive
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -85,8 +80,8 @@ stackhpc_pulp_repository_deb_repos:
   # Third-party repositories
   - name: Docker CE for Ubuntu
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu/{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -193,8 +188,8 @@ stackhpc_pulp_repository_rpm_repos:
   # Base CentOS 8 Stream repositories
   - name: CentOS Stream 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/AppStream/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -202,8 +197,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -211,8 +206,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/extras/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -222,40 +217,40 @@ stackhpc_pulp_repository_rpm_repos:
   # Base Rocky 8 repositories
   - name: Rocky Linux 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - NFV
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - PowerTools
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
@@ -264,8 +259,8 @@ stackhpc_pulp_repository_rpm_repos:
   # EPEL repositories
   - name: Extra Packages for Enterprise Linux 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_content_only
@@ -273,8 +268,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: Extra Packages for Enterprise Linux Modular 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -284,8 +279,8 @@ stackhpc_pulp_repository_rpm_repos:
   # Third-party repositories
   - name: Docker CE for CentOS 8
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/8/x86_64/stable/{{ stackhpc_pulp_repo_docker_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete

From 7437660f99595b3b93ae571ba2b8e5ee589b862a Mon Sep 17 00:00:00 2001
From: Matt Anson <matta@stackhpc.com>
Date: Tue, 6 Dec 2022 11:22:54 +0000
Subject: [PATCH 15/20] Set Pulp 3.21 init: false

---
 etc/kayobe/seed.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml
index 78cb92e79d..33e2637ac7 100644
--- a/etc/kayobe/seed.yml
+++ b/etc/kayobe/seed.yml
@@ -108,6 +108,9 @@ seed_pulp_container:
     post: "{{ kayobe_config_path }}/containers/pulp/post.yml"
     tag: "3.21"
     network_mode: host
+    # Override deploy_containers_defaults.init == true to ensure
+    # s6-overlay-suexec starts as pid 1
+    init: false
     volumes:
       - /opt/kayobe/containers/pulp:/etc/pulp
       - pulp_storage:/var/lib/pulp

From ec01a76b0616331ead0c8034696cc0b8cf63d765 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 5 Dec 2022 17:08:35 +0000
Subject: [PATCH 16/20] Adding support for pulp RBAC

---
 README.rst                                    | 37 +++++++---
 .../certs/ark.stackhpc.com/client-cert.pem    |  0
 .../certs/ark.stackhpc.com/client-key.pem     |  0
 etc/kayobe/ansible/requirements.yml           |  4 +-
 etc/kayobe/pulp.yml                           | 67 +++++++++----------
 5 files changed, 60 insertions(+), 48 deletions(-)
 delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem
 delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem

diff --git a/README.rst b/README.rst
index 304a8edbc7..21f7d67537 100644
--- a/README.rst
+++ b/README.rst
@@ -13,7 +13,7 @@ StackHPC provides packages and container images for OpenStack via `Ark
 
 Deployments should use a local `Pulp <https://pulpproject.org/>`__ repository
 server to synchronise content from Ark and serve it locally. Access to the
-repositories on Ark is controlled via X.509 certificates issued by StackHPC.
+repositories on Ark is controlled via user accounts issued by StackHPC.
 
 This configuration is a base, and should be merged with any existing Kayobe
 configuration. It currently provides the following:
@@ -103,20 +103,14 @@ Pulp startup.
 StackHPC Ark
 ------------
 
-The container image registry credentials issued by StackHPC should be
-configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the
-password:
+The Ark pulp credentials issued by StackHPC should be configured in
+``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password:
 
 .. code-block:: yaml
 
    stackhpc_release_pulp_username: <username>
    stackhpc_release_pulp_password: <password>
 
-The client certificate and private key issued by StackHPC should be stored in
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively,
-with the private key encrypted via Ansible Vault.
-
 The distribution name for the environment should be configured as either
 ``development`` or ``production`` via ``stackhpc_repo_distribution`` in
 ``etc/kayobe/stackhpc.yml``.
@@ -216,6 +210,31 @@ with the push repository using the pulp CLI:
     Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/
     .Done.
 
+HTTP Error 404: Not Found 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your login credentials are incorrect, or lack the required permissions, 
+you will see a 404 error during ``pulp-repo-sync.yml``:
+
+.. code-block:: console
+    TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] ****************************************************************************************************************************************
+    An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435'))
+    failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false 
+      ansible_loop_var: item
+      item:
+        name: centos-stream-8-baseos-development
+        policy: on_demand
+        proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287
+        remote_password: <password>
+        remote_username: <username>
+        required: true
+        state: present
+        sync_policy: mirror_complete
+        url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435
+      msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) '''
+The issue can be rectified by updating the ``stackhpc_release_pulp_username``
+and ``stackhpc_release_pulp_password`` variables
+
 Environments
 ============
 
diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 8293d49dc4..0ced4d5277 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -1,6 +1,4 @@
 ---
 collections:
   - name: stackhpc.pulp
-    version: 0.4.0
-  - name: pulp.squeezer
-    version: 0.0.11
+    version: 0.4.1
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index cdc62c85f9..d6017895ef 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -19,15 +19,10 @@ pulp_proxy_url: "{{ omit }}"
 # Base URL of the StackHPC Pulp service.
 stackhpc_release_pulp_url: "https://ark.stackhpc.com"
 
-# Credentials used to access the StackHPC Ark container image registry.
+# Credentials used to access the StackHPC Ark pulp server.
 stackhpc_release_pulp_username:
 stackhpc_release_pulp_password:
 
-# Client certificates used to access StackHPC Ark repositories.
-# They are trusted by the 'release' cert guard's CA.
-stackhpc_release_pulp_client_cert: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-cert.pem') | trim }}"
-stackhpc_release_pulp_client_key: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-key.pem') | trim }}"
-
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
@@ -44,8 +39,8 @@ stackhpc_pulp_repository_deb_repos:
   # Base Ubuntu Focal repositories
   - name: Ubuntu focal
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal/{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -57,8 +52,8 @@ stackhpc_pulp_repository_deb_repos:
 
   - name: Ubuntu focal security
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal-security/{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -71,8 +66,8 @@ stackhpc_pulp_repository_deb_repos:
   # Ubuntu Cloud Archive (UCA) repositories
   - name: Ubuntu Cloud Archive
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -85,8 +80,8 @@ stackhpc_pulp_repository_deb_repos:
   # Third-party repositories
   - name: Docker CE for Ubuntu
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu/{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -193,8 +188,8 @@ stackhpc_pulp_repository_rpm_repos:
   # Base CentOS 8 Stream repositories
   - name: CentOS Stream 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/AppStream/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -202,8 +197,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -211,8 +206,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/extras/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -222,40 +217,40 @@ stackhpc_pulp_repository_rpm_repos:
   # Base Rocky 8 repositories
   - name: Rocky Linux 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - NFV
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - PowerTools
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
@@ -264,8 +259,8 @@ stackhpc_pulp_repository_rpm_repos:
   # EPEL repositories
   - name: Extra Packages for Enterprise Linux 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_content_only
@@ -273,8 +268,8 @@ stackhpc_pulp_repository_rpm_repos:
     required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: Extra Packages for Enterprise Linux Modular 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -284,8 +279,8 @@ stackhpc_pulp_repository_rpm_repos:
   # Third-party repositories
   - name: Docker CE for CentOS 8
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/8/x86_64/stable/{{ stackhpc_pulp_repo_docker_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete

From 60f8588d5db698081a337d4fc32546855a2e560e Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Mon, 5 Dec 2022 17:29:18 +0000
Subject: [PATCH 17/20] updating docs

---
 doc/source/configuration/release-train.rst | 37 +++++++++++++++++-----
 etc/kayobe/ansible/advise-run.yml          |  2 +-
 etc/kayobe/ansible/requirements.yml        |  1 -
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst
index 8a25f0653d..39bb846c33 100644
--- a/doc/source/configuration/release-train.rst
+++ b/doc/source/configuration/release-train.rst
@@ -7,7 +7,7 @@ StackHPC provides packages and container images for OpenStack via `Ark
 
 Deployments should use a local `Pulp <https://pulpproject.org/>`__ repository
 server to synchronise content from Ark and serve it locally. Access to the
-repositories on Ark is controlled via X.509 certificates issued by StackHPC.
+repositories on Ark is controlled via user accounts issued by StackHPC.
 
 This configuration is a base, and should be merged with any existing Kayobe
 configuration. It currently provides the following:
@@ -41,19 +41,14 @@ Pulp startup.
 StackHPC Ark
 ------------
 
-The container image registry credentials issued by StackHPC should be
-configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the
-password:
+The Ark pulp credentials issued by StackHPC should be configured in
+``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password:
 
 .. code-block:: yaml
 
    stackhpc_release_pulp_username: <username>
    stackhpc_release_pulp_password: <password>
 
-The client certificate and private key issued by StackHPC should be stored in
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively,
-with the private key encrypted via Ansible Vault.
 
 The distribution name for the environment should be configured as either
 ``development`` or ``production`` via ``stackhpc_repo_distribution`` in
@@ -153,3 +148,29 @@ with the push repository using the pulp CLI:
     (venv-pulp) [stack@seed ~]$ pulp --base-url http://<pulp server>:8080--username admin --password <password> container distribution destroy --name stackhpc/centos-source-prometheus-jiralert
     Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/
     .Done.
+
+HTTP Error 404: Not Found 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your login credentials are incorrect, or lack the required permissions, 
+you will see a 404 error during ``pulp-repo-sync.yml``:
+
+.. code-block:: console
+    TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] ****************************************************************************************************************************************
+    An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435'))
+    failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false 
+      ansible_loop_var: item
+      item:
+        name: centos-stream-8-baseos-development
+        policy: on_demand
+        proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287
+        remote_password: <password>
+        remote_username: <username>
+        required: true
+        state: present
+        sync_policy: mirror_complete
+        url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435
+      msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) '''
+
+The issue can be rectified by updating the ``stackhpc_release_pulp_username``
+and ``stackhpc_release_pulp_password`` variables
diff --git a/etc/kayobe/ansible/advise-run.yml b/etc/kayobe/ansible/advise-run.yml
index 09440aeb95..d0ad2eee16 100644
--- a/etc/kayobe/ansible/advise-run.yml
+++ b/etc/kayobe/ansible/advise-run.yml
@@ -8,7 +8,7 @@
     venv: "~/venvs/advise-review"
     input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
     output_dir: "{{ lookup('env', 'PWD') }}/review"
-    advise_pattern: ".*.eval" # Uses regex
+    advise_pattern: ".*.eval"  # Uses regex
   tasks:
     - name: Install dependencies
       pip:
diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml
index 2ab9ab6617..77aaa9f993 100644
--- a/etc/kayobe/ansible/requirements.yml
+++ b/etc/kayobe/ansible/requirements.yml
@@ -6,4 +6,3 @@ collections:
     version: 0.4.1
 roles:
   - src: stackhpc.vxlan
-

From 25d50f722b90ddf1dd895888e48ada825c95cc91 Mon Sep 17 00:00:00 2001
From: Pierre Riteau <pierre@stackhpc.com>
Date: Fri, 11 Nov 2022 09:31:24 +0100
Subject: [PATCH 18/20] Wait longer before raising Elasticsearch alerts

---
 etc/kayobe/kolla/config/prometheus/elasticsearch.rules | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules
index 974bf4e998..42a196b9ad 100644
--- a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules
+++ b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules
@@ -44,7 +44,7 @@ groups:
 
   - alert: ElasticsearchClusterRed
     expr: elasticsearch_cluster_health_status{color="red"} == 1
-    for: 0m
+    for: 5m
     labels:
       severity: critical
     annotations:
@@ -53,7 +53,7 @@ groups:
 
   - alert: ElasticsearchClusterYellow
     expr: elasticsearch_cluster_health_status{color="yellow"} == 1
-    for: 5m
+    for: 15m
     labels:
       severity: warning
     annotations:
@@ -80,7 +80,7 @@ groups:
 
   - alert: ElasticsearchUnassignedShards
     expr: elasticsearch_cluster_health_unassigned_shards > 0
-    for: 0m
+    for: 5m
     labels:
       severity: critical
     annotations:

From 3933e4520ba512b5bf095a28b791c0bac12c5dd0 Mon Sep 17 00:00:00 2001
From: Matt <mattc@stackhpc.com>
Date: Tue, 6 Dec 2022 14:41:46 +0000
Subject: [PATCH 19/20] Hammer playbook for rabbitmq

---
 etc/kayobe/ansible/rabbitmq-reset.yml | 57 +++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 etc/kayobe/ansible/rabbitmq-reset.yml

diff --git a/etc/kayobe/ansible/rabbitmq-reset.yml b/etc/kayobe/ansible/rabbitmq-reset.yml
new file mode 100644
index 0000000000..df6d0c4cae
--- /dev/null
+++ b/etc/kayobe/ansible/rabbitmq-reset.yml
@@ -0,0 +1,57 @@
+---
+# Reset a broken RabbitMQ cluster.
+# Also restarts OpenStack services which may be broken.
+
+- name: Reset RabbitMQ
+  hosts: controllers
+  become: True
+  gather_facts: no
+  tags:
+    - rabbitmq-reset
+  vars:
+    - container_name: rabbitmq
+  tasks:
+    - name: Inspect the {{ container_name }} container
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }}"
+      register: inspection
+
+    - name: Ensure the {{ container_name }} container is running
+      command: "docker start {{ container_name }}"
+      when: inspection.stdout == 'false'
+
+    - name: Wait for the {{ container_name }} container to reach state 'Running'
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }}"
+      register: result
+      until: result.stdout == 'true'
+      retries: 10
+      delay: 6
+
+    - name: Wait for the rabbitmq node to automatically start on container start
+      command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'"
+      when: inspection.stdout == 'false'
+
+    - name: Stop app
+      command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl stop_app'"
+
+    - name: Force reset app
+      command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl force_reset'"
+
+    - name: Start app
+      command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl start_app'"
+
+    - name: Wait for all nodes to join the cluster
+      command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl await_online_nodes {{ groups['controllers'] | length }}'"
+
+- name: Restart OpenStack services
+  hosts: controllers:compute
+  become: true
+  gather_facts: no
+  tags:
+    - restart-openstack
+  tasks:
+    # The following services can have problems if the cluster gets broken.
+    - name: Restart OpenStack services
+      shell: >-
+        docker ps -a | egrep '(cinder|heat|ironic|keystone|magnum|neutron|nova)' | awk '{ print $NF }' | xargs docker restart

From 3f249764ead1e3fa2e5accb0e30fc3bde749cae9 Mon Sep 17 00:00:00 2001
From: Matt Anson <matta@stackhpc.com>
Date: Tue, 6 Dec 2022 16:09:14 +0000
Subject: [PATCH 20/20] Add Rocky minor version to Pulp URL in CI

---
 etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
index 7607ce3d1f..4ea3def7ec 100644
--- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml
@@ -41,9 +41,16 @@ stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_
 stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
 stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
 stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
-stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_6_baseos_version }}"
-stackhpc_repo_rocky_appstream_version: "{{ stackhpc_pulp_repo_rocky_8_6_appstream_version }}"
-stackhpc_repo_rocky_extras_version: "{{ stackhpc_pulp_repo_rocky_8_6_extras_version }}"
+## Use derived vars from etc/kayobe/pulp.yml to switch between
+## minor Rocky versions using stackhpc_pulp_repo_rocky_8_minor_version
+stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_baseos_version }}"
+stackhpc_repo_rocky_appstream_version: "{{ stackhpc_pulp_repo_rocky_8_appstream_version }}"
+stackhpc_repo_rocky_extras_version: "{{ stackhpc_pulp_repo_rocky_8_extras_version }}"
+
+# Rocky-and-CI-specific Pulp urls
+stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}/"
+stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}/"
+stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}/"
 
 # Host and port of container registry.
 # Push built images to the development Pulp service registry.