From ea77a86f795c7ac4d5d1aa37adf3ed94fe7937bf Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 15 Dec 2021 17:16:04 +0000 Subject: [PATCH 1/9] Fix Pulp repository promotion * Distribution promotion should not specify the repository * Production should be promoted from development, not staging (cherry picked from commit b2134243d0f65c9ac881bf13fe3e67e465ea91fb) --- etc/kayobe/pulp.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 1f3eddbd32..8305d17f56 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -118,36 +118,30 @@ stackhpc_pulp_distribution_rpm_development: stackhpc_pulp_distribution_rpm_production: # Base CentOS 8 Stream repositories - name: "centos-stream-8-baseos-production" - repository: CentOS Stream 8 - BaseOS base_path: "centos/8-stream/BaseOS/x86_64/os/production" - distribution: "centos-stream-8-baseos-staging" + distribution: "centos-stream-8-baseos-development" state: present - name: "centos-stream-8-appstream-production" - repository: CentOS Stream 8 - AppStream base_path: "centos/8-stream/AppStream/x86_64/os/production" - distribution: "centos-stream-8-appstream-staging" + distribution: "centos-stream-8-appstream-development" state: present - name: "centos-stream-8-extras-production" - repository: CentOS Stream 8 - Extras base_path: "centos/8-stream/extras/x86_64/os/production" - distribution: "centos-stream-8-extras-staging" + distribution: "centos-stream-8-extras-development" state: present # EPEL repositories - name: "extra-packages-for-enterprise-linux-8-x86_64-production" - repository: Extra Packages for Enterprise Linux 8 - x86_64 base_path: "epel/8/Everything/x86_64/production" - distribution: "extra-packages-for-enterprise-linux-8-x86_64-staging" + distribution: "extra-packages-for-enterprise-linux-8-x86_64-development" state: present - name: "extra-packages-for-enterprise-linux-modular-8-x86_64-production" - repository: Extra Packages for Enterprise Linux Modular 8 - x86_64 base_path: "epel/8/Modular/x86_64/production" - distribution: "extra-packages-for-enterprise-linux-modular-8-x86_64-staging" + distribution: "extra-packages-for-enterprise-linux-modular-8-x86_64-development" state: present # Third-party repositories - name: "docker-ce-for-centos-8-production" - repository: Docker CE for CentOS 8 base_path: "docker-ce/centos/8/x86_64/stable/production" - distribution: "docker-ce-for-centos-8-staging" + distribution: "docker-ce-for-centos-8-development" state: present From f49cb4e8efa0752e0cc438bb170bc93d073322a4 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 8 Dec 2021 14:31:57 +0000 Subject: [PATCH 2/9] Deploy a local pulp server on the seed Co-authored-by: Michal Nasiadka Co-authored-by: Bartosz Bezak (cherry picked from commit 7ef439a1a33597efa232876f1d429abb8140feb0) --- etc/kayobe/containers/pulp/pre.yml | 21 +++++++++++++++++++++ etc/kayobe/containers/pulp/settings.py | 4 ++++ etc/kayobe/seed.yml | 14 +++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/containers/pulp/pre.yml create mode 100644 etc/kayobe/containers/pulp/settings.py diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml new file mode 100644 index 0000000000..22d9990232 --- /dev/null +++ b/etc/kayobe/containers/pulp/pre.yml @@ -0,0 +1,21 @@ +--- +- name: Ensure /opt/kayobe/containers/pulp exists + file: + path: "/opt/kayobe/containers/pulp" + state: directory + become: true + +- name: Ensure required Docker volumes exist + docker_volume: + name: "{{ item }}" + loop: + - pulp_containers + - pulp_pgsql + - pulp_storage + +- name: Copy modified settings.py + template: + src: "{{ kayobe_config_path }}/containers/pulp/settings.py" + dest: /opt/kayobe/containers/pulp/settings.py + mode: 0644 + become: true diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py new file mode 100644 index 0000000000..d07798b3a5 --- /dev/null +++ b/etc/kayobe/containers/pulp/settings.py @@ -0,0 +1,4 @@ +CONTENT_ORIGIN='http://{{ ansible_fqdn }}' +ANSIBLE_API_HOSTNAME='http://{{ ansible_fqdn }}' +ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_fqdn }}/pulp/content' +TOKEN_AUTH_DISABLED=True diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index e0b5e0d004..c2d60b6010 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -95,7 +95,19 @@ # pre: "{{ kayobe_config_path }}/containers/squid/pre.yml" # post: "{{ kayobe_config_path }}/containers/squid/post.yml" # -#seed_containers: +seed_containers: + pulp: + name: pulp + image: pulp/pulp + pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" + tag: "3.16" + network_mode: host + volumes: + - /opt/kayobe/containers/pulp:/etc/pulp + - pulp_storage:/var/lib/pulp + - pulp_pgsql:/var/lib/pgsql + - pulp_containers:/var/lib/containers + restart_policy: unless-stopped ############################################################################### # Dummy variable to allow Ansible to accept this file. From dedb0cfefe56bf3cba554eca4888c402997090ec Mon Sep 17 00:00:00 2001 From: Isaac Prior Date: Fri, 10 Dec 2021 11:50:15 +0000 Subject: [PATCH 3/9] Use full var paths for settings template (cherry picked from commit 527b5874eb9f05271d15d5876e0a4fa68790440b) --- etc/kayobe/containers/pulp/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py index d07798b3a5..c304548454 100644 --- a/etc/kayobe/containers/pulp/settings.py +++ b/etc/kayobe/containers/pulp/settings.py @@ -1,4 +1,4 @@ -CONTENT_ORIGIN='http://{{ ansible_fqdn }}' -ANSIBLE_API_HOSTNAME='http://{{ ansible_fqdn }}' -ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_fqdn }}/pulp/content' +CONTENT_ORIGIN='http://{{ ansible_facts.fqdn }}' +ANSIBLE_API_HOSTNAME='http://{{ ansible_facts.fqdn }}' +ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_facts.fqdn }}/pulp/content' TOKEN_AUTH_DISABLED=True From 479a3cc53a7dc5a02d31f186d3fe3518f55c423a Mon Sep 17 00:00:00 2001 From: Isaac Prior Date: Fri, 10 Dec 2021 12:23:52 +0000 Subject: [PATCH 4/9] Link to kayobe docs for pulp deployment mechanism (cherry picked from commit 1cb6e8807d971fbf35a76b3bc279920be88ce512) --- README.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 31922a33c3..9543604930 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,7 @@ repositories on Ark is controlled via X.509 certificates issued by StackHPC. This configuration is a base, and should be merged with any existing Kayobe configuration. It currently provides the following: +* Configuration to deploy a local Pulp service * Pulp repository definitions for CentOS Stream 8 * Playbooks to synchronise a local Pulp service with Ark * Configuration to use the local Pulp repository mirrors on control plane hosts @@ -84,9 +85,14 @@ The distribution name for the environment should be configured as either Usage ===== -Several custom playbooks are provided in ``etc/kayobe/ansible/``. See the -Kayobe `custom playbook documentation -`__ +The local Pulp service will be deployed as a `Seed custom container +`__ +on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. + +The following custom playbooks are provided in ``etc/kayobe/ansible/``: + +See the Kayobe `custom playbook documentation +`__ for information on how to run them. * ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will From 953da3ca6a4794c956cf43df663e1c0c146f8eb6 Mon Sep 17 00:00:00 2001 From: Isaac Prior Date: Fri, 10 Dec 2021 17:25:54 +0000 Subject: [PATCH 5/9] Set the pulp api admin password (cherry picked from commit 3e477879f0e64eccd65e3ce7f49a39d393da435d) --- README.rst | 18 ++++++++++++------ etc/kayobe/containers/pulp/post.yml | 11 +++++++++++ etc/kayobe/pulp.yml | 7 +++++-- etc/kayobe/seed.yml | 1 + 4 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 etc/kayobe/containers/pulp/post.yml diff --git a/README.rst b/README.rst index 9543604930..90ca6664e4 100644 --- a/README.rst +++ b/README.rst @@ -65,18 +65,24 @@ Configuration ============= The URL and credentials of the local Pulp server should be configured in -``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password: +``etc/kayobe/pulp.yml`` in advance of deployment, using Ansible Vault +to encrypt the password: .. code-block:: yaml - pulp_url: + pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:8080" pulp_username: admin pulp_password: -The client certificate and key issued by StackHPC should be stored in -``certs/ark.stackhpc.com/client-cert.pem`` and -``certs/ark.stackhpc.com/client-key.pem``, respectively, with the private key -encrypted via Ansible Vault. +This is used to configure `Basic Auth for the Pulp API +`__. +Note that ``pulp_username`` is currently unused as only `admin` is supported. +``pulp_password`` is used to automatically set the admin password. + +The client certificate and private key issued by StackHPC should be stored in +``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and +``etc/kayobe/ansible/certs/ark.stackhpc.com/client-key.pem``, respectively, +with the private key encrypted via Ansible Vault. The distribution name for the environment should be configured as either ``development`` or ``production`` via ``stackhpc_repo_distribution`` in diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml new file mode 100644 index 0000000000..855260d507 --- /dev/null +++ b/etc/kayobe/containers/pulp/post.yml @@ -0,0 +1,11 @@ +--- +- name: Set the pulp api password + become: true + command: >- + docker exec -u root {{ seed_containers.pulp.name }} + bash -c + 'pulpcore-manager reset-admin-password -p {{ pulp_password }}' + no_log: true + register: pulp_manager_result + failed_when: + - "'Successfully set password' not in pulp_manager_result.stdout" diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 8305d17f56..5cc6983ba8 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -2,11 +2,14 @@ ############################################################################### # Local Pulp access credentials -pulp_url: http://localhost:8080 +# Base URL of the local Pulp service. +# Default uses the seed node's IP on the admin network. +pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:8080" + +# Credentials used to access the local Pulp REST API. pulp_username: admin pulp_password: - ############################################################################### # StackHPC Pulp server diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index c2d60b6010..cb40649541 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -100,6 +100,7 @@ seed_containers: name: pulp image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" + post: "{{ kayobe_config_path }}/containers/pulp/post.yml" tag: "3.16" network_mode: host volumes: From c0bc6a5a719ec298f2f8e4127f23073f145fdeb8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 14 Dec 2021 12:08:11 +0000 Subject: [PATCH 6/9] Wait for pulp to become ready before setting the password (cherry picked from commit 5d7378b1a2be406b7e1e212c2b70bbd9bb31c6a0) --- etc/kayobe/containers/pulp/post.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml index 855260d507..0548b99260 100644 --- a/etc/kayobe/containers/pulp/post.yml +++ b/etc/kayobe/containers/pulp/post.yml @@ -1,5 +1,13 @@ --- -- name: Set the pulp api password +- name: Wait for Pulp to become ready + uri: + url: "{{ pulp_url }}/pulp/api/v3/status/" + register: pulp_status + until: pulp_status is success + retries: 30 + delay: 2 + +- name: Set the Pulp admin password become: true command: >- docker exec -u root {{ seed_containers.pulp.name }} From a3fe8a8a08db023e00d19cab79ed8623876b1e5e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 14 Dec 2021 12:59:44 +0000 Subject: [PATCH 7/9] Use secrets.yml to set Pulp admin password, update readme (cherry picked from commit 736562d2343dc2ad667ba91e6efbdfdbc2470358) --- README.rst | 25 ++++++++++++------------- etc/kayobe/pulp.yml | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 90ca6664e4..be72932d40 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ repositories on Ark is controlled via X.509 certificates issued by StackHPC. This configuration is a base, and should be merged with any existing Kayobe configuration. It currently provides the following: -* Configuration to deploy a local Pulp service +* Configuration to deploy a local Pulp service as a container on the seed * Pulp repository definitions for CentOS Stream 8 * Playbooks to synchronise a local Pulp service with Ark * Configuration to use the local Pulp repository mirrors on control plane hosts @@ -64,20 +64,19 @@ need to merge the changes in this repository into your repository. Configuration ============= -The URL and credentials of the local Pulp server should be configured in -``etc/kayobe/pulp.yml`` in advance of deployment, using Ansible Vault -to encrypt the password: +Local Pulp server +----------------- -.. code-block:: yaml +The URL and credentials of the local Pulp server are configured in +``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and +``pulp_password``. In most cases, the default values should be sufficient. +An admin password must be generated and set as the value of a +``secrets_pulp_password`` variable, typically in an Ansible Vault encrypted +``etc/kayobe/secrets.yml`` file. This password will be automatically set on +Pulp startup. - pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:8080" - pulp_username: admin - pulp_password: - -This is used to configure `Basic Auth for the Pulp API -`__. -Note that ``pulp_username`` is currently unused as only `admin` is supported. -``pulp_password`` is used to automatically set the admin password. +StackHPC Ark +------------ The client certificate and private key issued by StackHPC should be stored in ``etc/kayobe/ansible/certs/ark.stackhpc.com/client-cert.pem`` and diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 5cc6983ba8..25e4639f43 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -8,7 +8,7 @@ pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:8080" # Credentials used to access the local Pulp REST API. pulp_username: admin -pulp_password: +pulp_password: "{{ secrets_pulp_password }}" ############################################################################### # StackHPC Pulp server From ee51a7d1907d89b1c95ffe1963e07eb45d880880 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 10 Dec 2021 17:27:41 +0000 Subject: [PATCH 8/9] readme: Fix name of pulp-repo-promote-production.yml playbook (cherry picked from commit f8e9a52ff365c6700c613a488b7da06caf459737) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index be72932d40..7aabf15aa1 100644 --- a/README.rst +++ b/README.rst @@ -110,7 +110,7 @@ for information on how to run them. (typically a development or staging environment). The new packages will not be available to cloud nodes using the ``production`` distribution until they have been promoted. -* ``pulp-repo-promote.yml``: Promote packages in the ``development`` +* ``pulp-repo-promote-production.yml``: Promote packages in the ``development`` distribution to the ``production`` distribution in the local Pulp. This will make all packages currently available to cloud nodes using the ``development`` distribution also available to cloud nodes using the From 52e0c343f2edee513675fdd794b845e989d6fc60 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 Dec 2021 16:39:19 +0000 Subject: [PATCH 9/9] Use port 80 for pulp_url due to host networking --- etc/kayobe/pulp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 25e4639f43..3a2c011c10 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -4,7 +4,7 @@ # Base URL of the local Pulp service. # Default uses the seed node's IP on the admin network. -pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:8080" +pulp_url: "http://{{ admin_oc_net_name | net_ip(groups['seed'][0]) }}:80" # Credentials used to access the local Pulp REST API. pulp_username: admin