diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 6831ad826..c831dd5f1 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -107,9 +107,9 @@ jobs: fi echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT - - name: Make sure dockerd is running and test Docker. + - name: Make sure dockerd is running and test Docker run: | - docker run --rm hello-world + docker ps - name: Output image tag id: image_tag diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml index a48072beb..8b308adf1 100644 --- a/.github/workflows/stackhpc-build-kayobe-image.yml +++ b/.github/workflows/stackhpc-build-kayobe-image.yml @@ -85,7 +85,7 @@ jobs: # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults # so that docker can run as a privileged user within the Kayobe image. - name: Build and push Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: file: ./.automation/docker/kayobe/Dockerfile context: . diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index e90316c39..6a8055ded 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -116,7 +116,7 @@ jobs: - name: Make sure dockerd is running and test Docker run: | - docker run --rm hello-world + docker ps - name: Install Kayobe run: | @@ -127,10 +127,11 @@ jobs: pip install -U pip && pip install ../src/kayobe - # Required for Docker registry login. Normally installed during host configure. + # Required for Pulp auth proxy deployment and Docker registry login. + # Normally installed during host configure. - name: Install Docker Python SDK run: | - pip install --user docker + sudo pip install docker - name: Configure localhost as a seed run: | @@ -141,11 +142,23 @@ jobs: localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 EOF + # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details. + # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the + # runner and dind containers. + - name: Deploy an authenticating package repository mirror proxy + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + - name: Build and push kolla overcloud images run: | args="${{ github.event.inputs.regexes }}" args="$args -e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=$KOLLA_TAG" + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" if ${{ inputs.push }} == 'true'; then args="$args --push" fi @@ -161,6 +174,7 @@ jobs: run: | args="-e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=$KOLLA_TAG" + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" if ${{ inputs.push }} == 'true'; then args="$args --push" fi diff --git a/.gitignore b/.gitignore index d83d0ce41..5891d3fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring # Tempest logs tempest-artifacts + +# Ansible Galaxy roles & collections +etc/kayobe/ansible/roles/*\.*/ +etc/kayobe/ansible/collections/ diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index bc4c373d6..f0a6f0ee9 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -101,6 +101,34 @@ Next, configure the host OS & services. kayobe seed host configure +.. _authenticating-pulp-proxy: + +Authenticating Pulp proxy +------------------------- + +If you are building against authenticated package repositories such as those in +`Ark `_, you will need to provide secure access to +the repositories without leaking credentials into the built images or their +metadata. This is typically not the case for a client-local Pulp, which +provides unauthenticated read-only access to the repositories on a trusted +network. + +Docker provides `build +secrets `_, but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +StackHPC Kayobe Configuration provides support for deploying an authenticating +Pulp proxy that injects an HTTP basic auth header into requests that it +proxies. Because this proxy bypasses Pulp's authentication, it must not be +exposed to any untrusted environment. + +To deploy the proxy: + +.. parsed-literal:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml + Building images =============== @@ -111,6 +139,9 @@ At this point you are ready to build and push some container images. kayobe seed container image build --push kayobe overcloud container image build --push +If using an :ref:`authenticating Pulp proxy `, +append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands. + The container images are tagged as |current_release|-. To use the new images, edit diff --git a/etc/kayobe/ansible/pulp-auth-proxy.yml b/etc/kayobe/ansible/pulp-auth-proxy.yml new file mode 100644 index 000000000..4cebbd386 --- /dev/null +++ b/etc/kayobe/ansible/pulp-auth-proxy.yml @@ -0,0 +1,14 @@ +--- +# See roles/pulp_auth_proxy/README.md for details. + +- name: Deploy Pulp auth proxy + hosts: container-image-builders + gather_facts: false + tasks: + - import_role: + name: pulp_auth_proxy + vars: + pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}" + pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}" + pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}" + pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy" diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 97086fd7e..e09ed5b85 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -11,7 +11,7 @@ collections: - name: stackhpc.hashicorp version: 2.4.0 - name: stackhpc.kayobe_workflows - version: 1.0.2 + version: 1.0.3 roles: - src: stackhpc.vxlan - name: ansible-lockdown.ubuntu22_cis diff --git a/etc/kayobe/ansible/reset-bls-entries.yml b/etc/kayobe/ansible/reset-bls-entries.yml old mode 100755 new mode 100644 diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md new file mode 100644 index 000000000..f14a5b2e8 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md @@ -0,0 +1,26 @@ +# Pulp Auth Proxy + +There is currently no practical, secure way to provide credentials for +accessing Ark's authenticated package repositories from within a Kolla build. +Docker provides [build +secrets](https://docs.docker.com/build/building/secrets/), but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +This role deploys an Nginx container that runs as a reverse proxy, injecting an +HTTP basic authentication header into requests. + +Because this proxy bypasses Pulp's authentication, it must not be exposed to +any untrusted environment. + +## Role variables + +* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to. +* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to. +* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to. +* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx + configuration. +* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to + listen. Default is `127.0.0.1`. +* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen. + Default is 80. diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml new file mode 100644 index 000000000..ae723565d --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml @@ -0,0 +1,7 @@ +--- +pulp_auth_proxy_url: +pulp_auth_proxy_username: +pulp_auth_proxy_password: +pulp_auth_proxy_conf_path: +pulp_auth_proxy_listen_ip: 127.0.0.1 +pulp_auth_proxy_listen_port: 80 diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml new file mode 100644 index 000000000..c15421510 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists" + ansible.builtin.file: + path: "{{ pulp_auth_proxy_conf_path }}" + state: directory + mode: 0700 + become: true + +- name: Ensure pulp_proxy.conf is templated + ansible.builtin.template: + src: pulp_proxy.conf.j2 + dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf" + mode: 0600 + become: true + register: pulp_proxy_conf + +- name: Ensure pulp_proxy container is running + community.docker.docker_container: + name: pulp_proxy + image: nginx:stable-alpine + ports: + - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80" + restart_policy: "no" + restart: "{{ pulp_proxy_conf is changed }}" + volumes: + - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro" diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 new file mode 100644 index 000000000..3d5a87ae7 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 @@ -0,0 +1,17 @@ +server { + listen {{ pulp_auth_proxy_listen_port }}; + server_name pulp_proxy; + location / { + proxy_pass {{ pulp_auth_proxy_url }}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }}; + # The important part: add basic auth header + proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}"; + proxy_pass_header Authorization; + # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027 + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2; + } +} diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml index 1893421c6..bb5cf5dca 100644 --- a/etc/kayobe/ansible/smartmon-tools.yml +++ b/etc/kayobe/ansible/smartmon-tools.yml @@ -2,7 +2,7 @@ - hosts: overcloud tasks: - - name: Ensure smartmon-tools, jq, nvme-cli and cron/cronie is installed + - name: Ensure smartmontools, jq, nvme-cli and cron/cronie are installed package: name: - smartmontools diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index e0c0e8de2..a017a69d7 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -40,7 +40,7 @@ resolv_is_managed: false # Host and port of a package repository mirror. # Build against the development Pulp service repositories. # Use Ark's package repositories to install packages. -stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}" stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}" diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 1d5481163..988ac221a 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -220,16 +220,21 @@ stackhpc_epel_9_repos: base_centos_repo_overrides_post_yum_list: "{{ stackhpc_rocky_9_repos + stackhpc_epel_9_repos + stackhpc_rocky_9_additional_repos + stackhpc_rocky_9_third_party_repos }}" stackhpc_yum_repos: "{{ stackhpc_rocky_9_repos }}" +# Apt sources.list entry prefix. +# If using an authenticating Pulp proxy we need to trust the repository because +# the certificate provided by the upstream repo will not match the proxy's IP. +stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}" + # List of base repositories for Ubuntu Jammy. stackhpc_ubuntu_jammy_base_repos: - - "deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy main universe" - - "deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main universe" - - "deb {{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main universe" - - "deb {{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main universe" # List of UCA repositories for Ubuntu Jammy. stackhpc_ubuntu_jammy_uca_repos: - - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} jammy-updates/{{ openstack_release }} main" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} jammy-updates/{{ openstack_release }} main" # List of repositories for Ubuntu Jammy. stackhpc_ubuntu_jammy_repos: "{{ stackhpc_ubuntu_jammy_base_repos + stackhpc_ubuntu_jammy_uca_repos }}" @@ -250,26 +255,16 @@ kolla_build_blocks: sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ - {% if stackhpc_repo_mirror_username is truthy %} - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \ - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \ - {% endif %} -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \ {% endif %} {% endfor %} {% else %} RUN \ rm /etc/apt/sources.list && \ - rm -f /etc/apt/auth.conf && \ - {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %} - {# We lack the ca-certificates package at this stage, so don't verify the CA #} + {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %} + {# We lack the ca-certificates package at this stage, so don't verify the CA initially #} echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \ {% endif %} - {% if stackhpc_repo_mirror_username is truthy %} - echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \ - echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \ - echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \ - {% endif %} {% for repo in stackhpc_ubuntu_jammy_base_repos %} echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ {% endif %} @@ -287,10 +282,6 @@ kolla_build_blocks: sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ - {% if stackhpc_repo_mirror_username is truthy %} - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \ - -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \ - {% endif %} -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \ {% endfor %} {% endif %} @@ -301,13 +292,7 @@ kolla_build_blocks: {% endif %} RUN \ rm /etc/apt/sources.list && \ - rm -f /etc/apt/auth.conf && \ rm -f /etc/apt/apt.conf.d/90no-verify-peer && \ - {% if stackhpc_repo_mirror_username is truthy %} - echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \ - echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \ - echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \ - {% endif %} {% for repo in stackhpc_ubuntu_jammy_repos %} echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ {% endif %} diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index a6d99d56d..2e2ee399f 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -8,6 +8,17 @@ stackhpc_repo_mirror_username: # Password of a package repository mirror. stackhpc_repo_mirror_password: +# Whether to use an authenticating reverse proxy to access the package +# repository mirror. This may be used when building container images, to avoid +# injecting package repository mirror credentials into the built images. See +# ansible/roles/pulp_auth_proxy/README.md for details. +stackhpc_repo_mirror_auth_proxy_enabled: false + +# URL of an authenticating reverse proxy used to access the package repository +# mirror. Used during container image builds when +# stackhpc_repo_mirror_auth_proxy_enabled is true. +stackhpc_repo_mirror_auth_proxy_url: "http://localhost" + # Distribution name. Either 'development' or 'production'. stackhpc_repo_distribution: "development" diff --git a/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml new file mode 100644 index 000000000..e9d54f989 --- /dev/null +++ b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an + authenticating proxy for Pulp. This can be used when building container + images to avoid leaking credentials for package repositories into the built + images or their metadata.