From ec01a76b0616331ead0c8034696cc0b8cf63d765 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 5 Dec 2022 17:08:35 +0000 Subject: [PATCH 1/4] Adding support for pulp RBAC --- README.rst | 37 +++++++--- .../certs/ark.stackhpc.com/client-cert.pem | 0 .../certs/ark.stackhpc.com/client-key.pem | 0 etc/kayobe/ansible/requirements.yml | 4 +- etc/kayobe/pulp.yml | 67 +++++++++---------- 5 files changed, 60 insertions(+), 48 deletions(-) delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem delete mode 100644 etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem diff --git a/README.rst b/README.rst index 304a8edbc..21f7d6753 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ StackHPC provides packages and container images for OpenStack via `Ark Deployments should use a local `Pulp `__ repository server to synchronise content from Ark and serve it locally. Access to the -repositories on Ark is controlled via X.509 certificates issued by StackHPC. +repositories on Ark is controlled via user accounts issued by StackHPC. This configuration is a base, and should be merged with any existing Kayobe configuration. It currently provides the following: @@ -103,20 +103,14 @@ Pulp startup. StackHPC Ark ------------ -The container image registry credentials issued by StackHPC should be -configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the -password: +The Ark pulp credentials issued by StackHPC should be configured in +``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password: .. code-block:: yaml stackhpc_release_pulp_username: stackhpc_release_pulp_password: -The client certificate and private key issued by StackHPC should be stored in -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and -``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, -with the private key encrypted via Ansible Vault. - The distribution name for the environment should be configured as either ``development`` or ``production`` via ``stackhpc_repo_distribution`` in ``etc/kayobe/stackhpc.yml``. @@ -216,6 +210,31 @@ with the push repository using the pulp CLI: Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ .Done. +HTTP Error 404: Not Found +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your login credentials are incorrect, or lack the required permissions, +you will see a 404 error during ``pulp-repo-sync.yml``: + +.. code-block:: console + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** + An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) + failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false + ansible_loop_var: item + item: + name: centos-stream-8-baseos-development + policy: on_demand + proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287 + remote_password: + remote_username: + required: true + state: present + sync_policy: mirror_complete + url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435 + msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) ''' +The issue can be rectified by updating the ``stackhpc_release_pulp_username`` +and ``stackhpc_release_pulp_password`` variables + Environments ============ diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem b/etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem deleted file mode 100644 index e69de29bb..000000000 diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 8293d49dc..0ced4d527 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,6 +1,4 @@ --- collections: - name: stackhpc.pulp - version: 0.4.0 - - name: pulp.squeezer - version: 0.0.11 + version: 0.4.1 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index cdc62c85f..d6017895e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -19,15 +19,10 @@ pulp_proxy_url: "{{ omit }}" # Base URL of the StackHPC Pulp service. stackhpc_release_pulp_url: "https://ark.stackhpc.com" -# Credentials used to access the StackHPC Ark container image registry. +# Credentials used to access the StackHPC Ark pulp server. stackhpc_release_pulp_username: stackhpc_release_pulp_password: -# Client certificates used to access StackHPC Ark repositories. -# They are trusted by the 'release' cert guard's CA. -stackhpc_release_pulp_client_cert: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-cert.pem') | trim }}" -stackhpc_release_pulp_client_key: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-key.pem') | trim }}" - # Content URL of the StackHPC Pulp service. stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content" @@ -44,8 +39,8 @@ stackhpc_pulp_repository_deb_repos: # Base Ubuntu Focal repositories - name: Ubuntu focal url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal/{{ stackhpc_pulp_repo_ubuntu_focal_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -57,8 +52,8 @@ stackhpc_pulp_repository_deb_repos: - name: Ubuntu focal security url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal-security/{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -71,8 +66,8 @@ stackhpc_pulp_repository_deb_repos: # Ubuntu Cloud Archive (UCA) repositories - name: Ubuntu Cloud Archive url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -85,8 +80,8 @@ stackhpc_pulp_repository_deb_repos: # Third-party repositories - name: Docker CE for Ubuntu url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu/{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" architectures: amd64 @@ -193,8 +188,8 @@ stackhpc_pulp_repository_rpm_repos: # Base CentOS 8 Stream repositories - name: CentOS Stream 8 - AppStream url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/AppStream/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -202,8 +197,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" - name: CentOS Stream 8 - BaseOS url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -211,8 +206,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}" - name: CentOS Stream 8 - Extras url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/extras/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -222,40 +217,40 @@ stackhpc_pulp_repository_rpm_repos: # Base Rocky 8 repositories - name: Rocky Linux 8 - AppStream url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - BaseOS url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - Extras url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - NFV url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}" - name: Rocky Linux 8 - PowerTools url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand sync_policy: mirror_complete state: present @@ -264,8 +259,8 @@ stackhpc_pulp_repository_rpm_repos: # EPEL repositories - name: Extra Packages for Enterprise Linux 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_content_only @@ -273,8 +268,8 @@ stackhpc_pulp_repository_rpm_repos: required: "{{ stackhpc_pulp_sync_el_8 | bool }}" - name: Extra Packages for Enterprise Linux Modular 8 - x86_64 url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete @@ -284,8 +279,8 @@ stackhpc_pulp_repository_rpm_repos: # Third-party repositories - name: Docker CE for CentOS 8 url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/8/x86_64/stable/{{ stackhpc_pulp_repo_docker_version }}" - client_cert: "{{ stackhpc_release_pulp_client_cert }}" - client_key: "{{ stackhpc_release_pulp_client_key }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" policy: on_demand proxy_url: "{{ pulp_proxy_url }}" sync_policy: mirror_complete From 3933e4520ba512b5bf095a28b791c0bac12c5dd0 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 6 Dec 2022 14:41:46 +0000 Subject: [PATCH 2/4] Hammer playbook for rabbitmq --- etc/kayobe/ansible/rabbitmq-reset.yml | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 etc/kayobe/ansible/rabbitmq-reset.yml diff --git a/etc/kayobe/ansible/rabbitmq-reset.yml b/etc/kayobe/ansible/rabbitmq-reset.yml new file mode 100644 index 000000000..df6d0c4ca --- /dev/null +++ b/etc/kayobe/ansible/rabbitmq-reset.yml @@ -0,0 +1,57 @@ +--- +# Reset a broken RabbitMQ cluster. +# Also restarts OpenStack services which may be broken. + +- name: Reset RabbitMQ + hosts: controllers + become: True + gather_facts: no + tags: + - rabbitmq-reset + vars: + - container_name: rabbitmq + tasks: + - name: Inspect the {{ container_name }} container + shell: + cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }}" + register: inspection + + - name: Ensure the {{ container_name }} container is running + command: "docker start {{ container_name }}" + when: inspection.stdout == 'false' + + - name: Wait for the {{ container_name }} container to reach state 'Running' + shell: + cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }}" + register: result + until: result.stdout == 'true' + retries: 10 + delay: 6 + + - name: Wait for the rabbitmq node to automatically start on container start + command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'" + when: inspection.stdout == 'false' + + - name: Stop app + command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl stop_app'" + + - name: Force reset app + command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl force_reset'" + + - name: Start app + command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl start_app'" + + - name: Wait for all nodes to join the cluster + command: "docker exec -it {{ container_name }} /bin/bash -c 'rabbitmqctl await_online_nodes {{ groups['controllers'] | length }}'" + +- name: Restart OpenStack services + hosts: controllers:compute + become: true + gather_facts: no + tags: + - restart-openstack + tasks: + # The following services can have problems if the cluster gets broken. + - name: Restart OpenStack services + shell: >- + docker ps -a | egrep '(cinder|heat|ironic|keystone|magnum|neutron|nova)' | awk '{ print $NF }' | xargs docker restart From 3f249764ead1e3fa2e5accb0e30fc3bde749cae9 Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 6 Dec 2022 16:09:14 +0000 Subject: [PATCH 3/4] Add Rocky minor version to Pulp URL in CI --- etc/kayobe/environments/ci-aio/stackhpc-ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml index 7607ce3d1..4ea3def7e 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -41,9 +41,16 @@ stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_ stackhpc_repo_ubuntu_focal_version: "{{ stackhpc_pulp_repo_ubuntu_focal_version }}" stackhpc_repo_ubuntu_focal_security_version: "{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}" stackhpc_repo_docker_ce_ubuntu_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}" -stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_6_baseos_version }}" -stackhpc_repo_rocky_appstream_version: "{{ stackhpc_pulp_repo_rocky_8_6_appstream_version }}" -stackhpc_repo_rocky_extras_version: "{{ stackhpc_pulp_repo_rocky_8_6_extras_version }}" +## Use derived vars from etc/kayobe/pulp.yml to switch between +## minor Rocky versions using stackhpc_pulp_repo_rocky_8_minor_version +stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_baseos_version }}" +stackhpc_repo_rocky_appstream_version: "{{ stackhpc_pulp_repo_rocky_8_appstream_version }}" +stackhpc_repo_rocky_extras_version: "{{ stackhpc_pulp_repo_rocky_8_extras_version }}" + +# Rocky-and-CI-specific Pulp urls +stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}/" +stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}/" +stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}/" # Host and port of container registry. # Push built images to the development Pulp service registry. From 61f26484758f012107a271c2716e0ac0bf139e10 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 7 Dec 2022 13:50:34 +0000 Subject: [PATCH 4/4] Fix release-train docs formatting --- doc/source/configuration/release-train.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst index 39bb846c3..04b583410 100644 --- a/doc/source/configuration/release-train.rst +++ b/doc/source/configuration/release-train.rst @@ -156,6 +156,7 @@ If your login credentials are incorrect, or lack the required permissions, you will see a 404 error during ``pulp-repo-sync.yml``: .. code-block:: console + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false