Switch from pulp certs to RBAC

doc/source/configuration/release-train.rst

@@ -7,7 +7,7 @@ StackHPC provides packages and container images for OpenStack via `Ark
 
 Deployments should use a local `Pulp <https://pulpproject.org/>`__ repository
 server to synchronise content from Ark and serve it locally. Access to the
-repositories on Ark is controlled via X.509 certificates issued by StackHPC.
+repositories on Ark is controlled via user accounts issued by StackHPC.
 
 This configuration is a base, and should be merged with any existing Kayobe
 configuration. It currently provides the following:
@@ -41,19 +41,14 @@ Pulp startup.
 StackHPC Ark
 ------------
 
-The container image registry credentials issued by StackHPC should be
-configured in ``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the
-password:
+The Ark pulp credentials issued by StackHPC should be configured in
+``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password:
 
 .. code-block:: yaml
 
    stackhpc_release_pulp_username: <username>
    stackhpc_release_pulp_password: <password>
 
-The client certificate and private key issued by StackHPC should be stored in
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and
-``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively,
-with the private key encrypted via Ansible Vault.
 
 The distribution name for the environment should be configured as either
 ``development`` or ``production`` via ``stackhpc_repo_distribution`` in
@@ -153,3 +148,29 @@ with the push repository using the pulp CLI:
     (venv-pulp) [stack@seed ~]$ pulp --base-url http://<pulp server>:8080--username admin --password <password> container distribution destroy --name stackhpc/centos-source-prometheus-jiralert
     Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/
     .Done.
+
+HTTP Error 404: Not Found 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your login credentials are incorrect, or lack the required permissions, 
+you will see a 404 error during ``pulp-repo-sync.yml``:
+
+.. code-block:: console
+    TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] ****************************************************************************************************************************************
+    An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435'))
+    failed: [localhost] (item=centos-stream-8-baseos-development) => changed=false 
+      ansible_loop_var: item
+      item:
+        name: centos-stream-8-baseos-development
+        policy: on_demand
+        proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287
+        remote_password: <password>
+        remote_username: <username>
+        required: true
+        state: present
+        sync_policy: mirror_complete
+        url: https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435
+      msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/centos/8-stream/BaseOS/x86_64/os/20211122T102435')) '''
+
+The issue can be rectified by updating the ``stackhpc_release_pulp_username``
+and ``stackhpc_release_pulp_password`` variables

etc/kayobe/ansible/advise-run.yml

@@ -0,0 +1,46 @@
+---
+- name: ADVise run
+  hosts: localhost
+  gather_facts: no
+  tags:
+    - advise
+  vars:
+    venv: "~/venvs/advise-review"
+    input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data"
+    output_dir: "{{ lookup('env', 'PWD') }}/review"
+    advise_pattern: ".*.eval"  # Uses regex
+  tasks:
+    - name: Install dependencies
+      pip:
+        virtualenv: "{{ venv }}"
+        name:
+          - git+https://github.com/stackhpc/ADVise
+        state: latest
+
+    - name: Create data directory
+      file:
+        path: '{{ output_dir }}/data'
+        state: directory
+
+    - name: Extract data
+      shell:
+        cmd: >
+          {{ venv }}/bin/m2-extract {{ input_dir }}/*.json --output_dir {{ output_dir }}/data
+
+    - name: Create review directory
+      file:
+        path: '{{ output_dir }}/results'
+        state: directory
+
+    - name: Process data
+      shell:
+        cmd: >
+          {{ venv }}/bin/advise-process
+          -I ipmi
+          -p '{{ output_dir }}/data/extra-hardware/{{ advise_pattern }}'
+          -o '{{ output_dir }}'
+
+    - name: Visualise data
+      command: >
+        {{ venv }}/bin/advise-visualise
+        --output_dir '{{ output_dir }}'

etc/kayobe/ansible/requirements.yml

@@ -3,6 +3,6 @@ collections:
   - name: stackhpc.cephadm
     version: 1.10.0
   - name: stackhpc.pulp
-    version: 0.3.0
+    version: 0.4.1
 roles:
   - src: stackhpc.vxlan

etc/kayobe/pulp.yml

@@ -19,15 +19,10 @@ pulp_proxy_url: "{{ omit }}"
 # Base URL of the StackHPC Pulp service.
 stackhpc_release_pulp_url: "https://ark.stackhpc.com"
 
-# Credentials used to access the StackHPC Ark container image registry.
+# Credentials used to access the StackHPC Ark pulp server.
 stackhpc_release_pulp_username:
 stackhpc_release_pulp_password:
 
-# Client certificates used to access StackHPC Ark repositories.
-# They are trusted by the 'release' cert guard's CA.
-stackhpc_release_pulp_client_cert: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-cert.pem') | trim }}"
-stackhpc_release_pulp_client_key: "{{ lookup('file', kayobe_config_path ~ '/ansible/certs/ark.stackhpc.com/client-key.pem') | trim }}"
-
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
@@ -44,8 +39,8 @@ stackhpc_pulp_repository_deb_repos:
   # Base Ubuntu Focal repositories
   - name: Ubuntu focal
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal/{{ stackhpc_pulp_repo_ubuntu_focal_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -57,8 +52,8 @@ stackhpc_pulp_repository_deb_repos:
 
   - name: Ubuntu focal security
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/focal-security/{{ stackhpc_pulp_repo_ubuntu_focal_security_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -71,8 +66,8 @@ stackhpc_pulp_repository_deb_repos:
   # Ubuntu Cloud Archive (UCA) repositories
   - name: Ubuntu Cloud Archive
     url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -85,8 +80,8 @@ stackhpc_pulp_repository_deb_repos:
   # Third-party repositories
   - name: Docker CE for Ubuntu
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu/{{ stackhpc_pulp_repo_docker_ce_ubuntu_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     architectures: amd64
@@ -193,26 +188,26 @@ stackhpc_pulp_repository_rpm_repos:
   # Base CentOS 8 Stream repositories
   - name: CentOS Stream 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/AppStream/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_centos_stream8 | bool }}"
   - name: CentOS Stream 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/extras/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -222,40 +217,40 @@ stackhpc_pulp_repository_rpm_repos:
   # Base Rocky 8 repositories
   - name: Rocky Linux 8 - AppStream
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_appstream_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - BaseOS
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_baseos_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - Extras
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_extras_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - NFV
     url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/nfv/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_nfv_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
     required: "{{ stackhpc_pulp_sync_rocky_8 | bool }}"
   - name: Rocky Linux 8 - PowerTools
-    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_6_powertools_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    url: "{{ stackhpc_release_pulp_content_url }}/rocky/8.{{ stackhpc_pulp_repo_rocky_8_minor_version }}/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_rocky_8_powertools_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     sync_policy: mirror_complete
     state: present
@@ -264,17 +259,17 @@ stackhpc_pulp_repository_rpm_repos:
   # EPEL repositories
   - name: Extra Packages for Enterprise Linux 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Everything/x86_64/{{ stackhpc_pulp_repo_epel_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_content_only
     state: present
     required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
   - name: Extra Packages for Enterprise Linux Modular 8 - x86_64
     url: "{{ stackhpc_release_pulp_content_url }}/epel/8/Modular/x86_64/{{ stackhpc_pulp_repo_epel_modular_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete
@@ -284,8 +279,8 @@ stackhpc_pulp_repository_rpm_repos:
   # Third-party repositories
   - name: Docker CE for CentOS 8
     url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/8/x86_64/stable/{{ stackhpc_pulp_repo_docker_version }}"
-    client_cert: "{{ stackhpc_release_pulp_client_cert }}"
-    client_key: "{{ stackhpc_release_pulp_client_key }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
     policy: on_demand
     proxy_url: "{{ pulp_proxy_url }}"
     sync_policy: mirror_complete

etc/kayobe/stackhpc.yml

@@ -104,15 +104,15 @@ stackhpc_repo_treasuredata_4_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/t
 stackhpc_repo_treasuredata_4_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 BaseOS
-stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}"
+stackhpc_repo_rocky_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/BaseOS/x86_64/os/{{ stackhpc_repo_rocky_baseos_version }}/"
 stackhpc_repo_rocky_baseos_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 AppStream
-stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}"
+stackhpc_repo_rocky_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/AppStream/x86_64/os/{{ stackhpc_repo_rocky_appstream_version }}/"
 stackhpc_repo_rocky_appstream_version: "{{ stackhpc_repo_distribution }}"
 
 # Rocky 8 extras
-stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8.6/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}"
+stackhpc_repo_rocky_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/8/extras/x86_64/os/{{ stackhpc_repo_rocky_extras_version }}/"
 stackhpc_repo_rocky_extras_version: "{{ stackhpc_repo_distribution }}"
 
 ###############################################################################