From 707d65b8bd6794e8000511eb6d65f28031140a01 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 12:01:58 +0000
Subject: [PATCH 1/6] Revert "Don't verify Apt repo CA initially when using
 HTTPS in container build"

This reverts commit f8947a99318d73827de2653d8027f69a6df31f3e.

This approach is not secure and leaves credentials in images.
---
 etc/kayobe/kolla.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 15407f00c..45c9b573c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -328,10 +328,6 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %}
-    {# We lack the ca-certificates package at this stage, so don't verify the CA #}
-        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
-    {% endif %}
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
@@ -369,7 +365,6 @@ kolla_build_blocks:
     RUN \
         rm /etc/apt/sources.list && \
         rm -f /etc/apt/auth.conf && \
-        rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
     {% if stackhpc_repo_mirror_username is truthy %}
         echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
         echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \

From 5d505223ef3524751f3aa2cedf096bac495d3f9c Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 12:02:20 +0000
Subject: [PATCH 2/6] Revert "Add package repository credentials to container
 image build"

This reverts commit 05e1995f6553e617f6bbc0712e9e1e7be18639c9.

This approach is not secure and leaves credentials in images.
---
 etc/kayobe/kolla.yml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 45c9b573c..082e9243b 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -317,22 +317,12 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -350,10 +340,6 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
     {% endif %}
@@ -364,12 +350,6 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}

From c794383d7b40a15ab5f009f69ba2891c89a82937 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:46:32 +0000
Subject: [PATCH 3/6] Add Ansible Galaxy roles and collections to .gitignore

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index d83d0ce41..5891d3fdd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 
 # Tempest logs
 tempest-artifacts
+
+# Ansible Galaxy roles & collections
+etc/kayobe/ansible/roles/*\.*/
+etc/kayobe/ansible/collections/

From 6dd2e7f5fb94188659572d5b2c44a4742966189a Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:44:55 +0000
Subject: [PATCH 4/6] Add a playbook to deploy an authenticating Pulp proxy

There is currently no practical, secure way to provide credentials for
accessing Ark's authenticated package repositories from within a Kolla
build.  Docker provides build secrets, but these must be explicitly
requested for each RUN statement, making them challenging to use in
Kolla.

This change adds a playbook that deploys an Nginx container that runs as
a reverse proxy, injecting an HTTP basic authentication header into
requests.

Because this proxy bypasses Pulp's authentication, it must not be
exposed to any untrusted environment.

[1] https://docs.docker.com/build/building/secrets/
---
 etc/kayobe/ansible/pulp-auth-proxy.yml        | 14 ++++++++++
 .../ansible/roles/pulp_auth_proxy/README.md   | 26 +++++++++++++++++++
 .../roles/pulp_auth_proxy/defaults/main.yml   |  7 +++++
 .../roles/pulp_auth_proxy/tasks/main.yml      | 26 +++++++++++++++++++
 .../templates/pulp_proxy.conf.j2              | 17 ++++++++++++
 5 files changed, 90 insertions(+)
 create mode 100644 etc/kayobe/ansible/pulp-auth-proxy.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
 create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2

diff --git a/etc/kayobe/ansible/pulp-auth-proxy.yml b/etc/kayobe/ansible/pulp-auth-proxy.yml
new file mode 100644
index 000000000..4cebbd386
--- /dev/null
+++ b/etc/kayobe/ansible/pulp-auth-proxy.yml
@@ -0,0 +1,14 @@
+---
+# See roles/pulp_auth_proxy/README.md for details.
+
+- name: Deploy Pulp auth proxy
+  hosts: container-image-builders
+  gather_facts: false
+  tasks:
+    - import_role:
+        name: pulp_auth_proxy
+      vars:
+        pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}"
+        pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}"
+        pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}"
+        pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy"
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
new file mode 100644
index 000000000..f14a5b2e8
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md
@@ -0,0 +1,26 @@
+# Pulp Auth Proxy
+
+There is currently no practical, secure way to provide credentials for
+accessing Ark's authenticated package repositories from within a Kolla build.
+Docker provides [build
+secrets](https://docs.docker.com/build/building/secrets/), but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+This role deploys an Nginx container that runs as a reverse proxy, injecting an
+HTTP basic authentication header into requests.
+
+Because this proxy bypasses Pulp's authentication, it must not be exposed to
+any untrusted environment.
+
+## Role variables
+
+* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx
+  configuration.
+* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to
+  listen. Default is `127.0.0.1`.
+* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen.
+  Default is 80.
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
new file mode 100644
index 000000000..ae723565d
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml
@@ -0,0 +1,7 @@
+---
+pulp_auth_proxy_url:
+pulp_auth_proxy_username:
+pulp_auth_proxy_password:
+pulp_auth_proxy_conf_path:
+pulp_auth_proxy_listen_ip: 127.0.0.1
+pulp_auth_proxy_listen_port: 80
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
new file mode 100644
index 000000000..c15421510
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml
@@ -0,0 +1,26 @@
+---
+- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists"
+  ansible.builtin.file:
+    path: "{{ pulp_auth_proxy_conf_path }}"
+    state: directory
+    mode: 0700
+  become: true
+
+- name: Ensure pulp_proxy.conf is templated
+  ansible.builtin.template:
+    src: pulp_proxy.conf.j2
+    dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf"
+    mode: 0600
+  become: true
+  register: pulp_proxy_conf
+
+- name: Ensure pulp_proxy container is running
+  community.docker.docker_container:
+    name: pulp_proxy
+    image: nginx:stable-alpine
+    ports:
+      - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80"
+    restart_policy: "no"
+    restart: "{{ pulp_proxy_conf is changed }}"
+    volumes:
+      - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro"
diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2
new file mode 100644
index 000000000..3d5a87ae7
--- /dev/null
+++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2
@@ -0,0 +1,17 @@
+server {
+    listen {{ pulp_auth_proxy_listen_port }};
+    server_name pulp_proxy;
+    location / {
+        proxy_pass {{ pulp_auth_proxy_url }};
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }};
+        # The important part: add basic auth header
+        proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}";
+        proxy_pass_header Authorization;
+        # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027
+        proxy_ssl_server_name on;
+        proxy_ssl_protocols TLSv1.2;
+    }
+}

From 9f940b7c82540863bbb957424df95978158a6d07 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 11:51:07 +0000
Subject: [PATCH 5/6] Use authenticating Pulp proxy during container image
 builds

This change uses the authenticating Pulp proxy added in
6320be9d01c38a8bf0c2b5950aebea8d4906ac64 to provide container image
builds access to Ark's authenticated package repositories without
injecting the credentials into the built images or their metadata.
---
 .../stackhpc-container-image-build.yml        | 18 +++++++++--
 .../contributor/environments/ci-builder.rst   | 31 +++++++++++++++++++
 .../environments/ci-builder/stackhpc-ci.yml   |  2 +-
 etc/kayobe/kolla.yml                          | 20 +++++++++---
 etc/kayobe/stackhpc.yml                       | 11 +++++++
 .../pulp-auth-proxy-24f0b31a4498441b.yaml     |  7 +++++
 6 files changed, 81 insertions(+), 8 deletions(-)
 create mode 100644 releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 296637775..717d6508f 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -136,10 +136,11 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      # Required for Docker registry login. Normally installed during host configure.
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          pip install --user docker
+          sudo pip install docker
 
       - name: Configure localhost as a seed
         run: |
@@ -150,11 +151,23 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -169,6 +182,7 @@ jobs:
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst
index 9df7426b5..ec14a9f1d 100644
--- a/doc/source/contributor/environments/ci-builder.rst
+++ b/doc/source/contributor/environments/ci-builder.rst
@@ -95,6 +95,34 @@ Next, configure the host OS & services.
 
    kayobe seed host configure
 
+.. _authenticating-pulp-proxy:
+
+Authenticating Pulp proxy
+-------------------------
+
+If you are building against authenticated package repositories such as those in
+`Ark <https://ark.stackhpc.com>`_, you will need to provide secure access to
+the repositories without leaking credentials into the built images or their
+metadata.  This is typically not the case for a client-local Pulp, which
+provides unauthenticated read-only access to the repositories on a trusted
+network.
+
+Docker provides `build
+secrets <https://docs.docker.com/build/building/secrets/>`_, but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+StackHPC Kayobe Configuration provides support for deploying an authenticating
+Pulp proxy that injects an HTTP basic auth header into requests that it
+proxies. Because this proxy bypasses Pulp's authentication, it must not be
+exposed to any untrusted environment.
+
+To deploy the proxy:
+
+.. parsed-literal::
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml
+
 Building images
 ===============
 
@@ -105,6 +133,9 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
+If using an :ref:`authenticating Pulp proxy <authenticating-pulp-proxy>`,
+append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands.
+
 The container images are tagged as |current_release|-<datetime>.
 
 To use the new images, edit
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 946759718..f31629357 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -43,7 +43,7 @@ resolv_is_managed: false
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
-stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
 
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 082e9243b..1a047f33c 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -287,16 +287,21 @@ base_centos_repo_overrides_post_yum_rocky_list: "{{ stackhpc_rocky_9_repos + sta
 base_centos_repo_overrides_post_yum_list: "{{ base_centos_repo_overrides_post_yum_centos_list if kolla_base_distro == 'centos' else base_centos_repo_overrides_post_yum_rocky_list }}"
 stackhpc_yum_repos: "{{ stackhpc_centos_stream_repos if kolla_base_distro == 'centos' else stackhpc_rocky_9_repos }}"
 
+# Apt sources.list entry prefix.
+# If using an authenticating Pulp proxy we need to trust the repository because
+# the certificate provided by the upstream repo will not match the proxy's IP.
+stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}"
+
 # List of base repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_base_repos:
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
 
 # List of UCA repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_uca_repos:
-  - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
 
 # List of repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_repos: "{{ stackhpc_ubuntu_focal_base_repos + stackhpc_ubuntu_focal_uca_repos }}"
@@ -323,6 +328,10 @@ kolla_build_blocks:
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
+    {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}
+    {# We lack the ca-certificates package at this stage, so don't verify the CA initially #}
+        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
+    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -350,6 +359,7 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
+        rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 21233ab34..513dd56d8 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -8,6 +8,17 @@ stackhpc_repo_mirror_username:
 # Password of a package repository mirror.
 stackhpc_repo_mirror_password:
 
+# Whether to use an authenticating reverse proxy to access the package
+# repository mirror.  This may be used when building container images, to avoid
+# injecting package repository mirror credentials into the built images.  See
+# ansible/roles/pulp_auth_proxy/README.md for details.
+stackhpc_repo_mirror_auth_proxy_enabled: false
+
+# URL of an authenticating reverse proxy used to access the package repository
+# mirror. Used during container image builds when
+# stackhpc_repo_mirror_auth_proxy_enabled is true.
+stackhpc_repo_mirror_auth_proxy_url: "http://localhost"
+
 # Distribution name. Either 'development' or 'production'.
 stackhpc_repo_distribution: "development"
 
diff --git a/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml
new file mode 100644
index 000000000..e9d54f989
--- /dev/null
+++ b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an
+    authenticating proxy for Pulp. This can be used when building container
+    images to avoid leaking credentials for package repositories into the built
+    images or their metadata.

From 74e87c51652aed8dc0f51bde92e22fa7c4224201 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Tue, 13 Feb 2024 14:22:13 +0000
Subject: [PATCH 6/6] CI: Avoid hitting Dockerhub rate limits

Running the hello-world container from Dockerhub at the beginning of
workflows can cause us to hit rate limits. Switch to a simple docker ps.
---
 .github/workflows/stackhpc-all-in-one.yml            | 4 ++--
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 26de98f66..704009412 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -86,9 +86,9 @@ jobs:
         with:
            submodules: true
 
-      - name: Make sure dockerd is running and test Docker.
+      - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Output image tag
         id: image_tag
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 717d6508f..b8afea93e 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -125,7 +125,7 @@ jobs:
 
       - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Install Kayobe
         run: |