From 1fe546e91cb01ff9b7277ff9ced108cf022ddc67 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 5 Apr 2023 15:30:37 +0100 Subject: [PATCH 1/2] Fix container image promotion (hopefully) This change modifies the way in which we promote container image tags. Previously, the stackhpc-dev container repository was configured as a remote, and the stackhpc repository synced with it, using include_tags to limit to a specific tag. Now, we use the "Add content" [1] API to add container tags from stackhpc-dev to stackhpc. Currently we're still promoting a single tag across all repositories, so a non-existent tag is not an error. This will change in future. [1] https://docs.pulpproject.org/pulp_container/restapi.html#tag/Repositories:-Container/operation/repositories_container_container_add --- ansible/dev-pulp-container-promote.yml | 21 ++++++ .../group_vars/all/dev-pulp-containers | 9 +-- .../roles/pulp-container-content/README.md | 49 +++++++++++++ .../pulp-container-content/defaults/main.yml | 6 ++ .../pulp-container-content/tasks/content.yml | 73 +++++++++++++++++++ .../pulp-container-content/tasks/main.yml | 4 + 6 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 ansible/roles/pulp-container-content/README.md create mode 100644 ansible/roles/pulp-container-content/defaults/main.yml create mode 100644 ansible/roles/pulp-container-content/tasks/content.yml create mode 100644 ansible/roles/pulp-container-content/tasks/main.yml diff --git a/ansible/dev-pulp-container-promote.yml b/ansible/dev-pulp-container-promote.yml index da67f1e1..e2cba475 100644 --- a/ansible/dev-pulp-container-promote.yml +++ b/ansible/dev-pulp-container-promote.yml @@ -33,3 +33,24 @@ pulp_username: "{{ dev_pulp_username }}" pulp_password: "{{ dev_pulp_password }}" pulp_distribution_container: "{{ dev_pulp_distribution_container_release }}" + + # Copy tags from stackhpc-dev to stackhpc repositories. + - import_role: + name: pulp-container-content + vars: + pulp_url: "{{ dev_pulp_url }}" + pulp_username: "{{ dev_pulp_username }}" + pulp_password: "{{ dev_pulp_password }}" + pulp_container_content: >- + {%- set contents = [] -%} + {%- for base_distro in kolla_base_distros -%} + {%- for image in kolla_container_images_filtered -%} + {%- if image not in kolla_unbuildable_images[base_distro]-%} + {%- set src_image_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%} + {%- set dest_image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%} + {%- set content = {"src_repo": src_image_repo, "src_is_push": true, "dest_repo": dest_image_repo, "tags": [dev_pulp_repository_container_promotion_tag]} -%} + {%- set _ = contents.append(content) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{ contents }} diff --git a/ansible/inventory/group_vars/all/dev-pulp-containers b/ansible/inventory/group_vars/all/dev-pulp-containers index 55604964..7856c20d 100644 --- a/ansible/inventory/group_vars/all/dev-pulp-containers +++ b/ansible/inventory/group_vars/all/dev-pulp-containers @@ -8,12 +8,6 @@ dev_release_pulp_registry_url: "{{ dev_pulp_url }}" # Common parameters for release image repositories. dev_pulp_repository_container_repo_release_common: - url: "{{ dev_release_pulp_registry_url }}" - include_tags: - - "{{ dev_pulp_repository_container_promotion_tag.strip() | mandatory }}" - policy: on_demand - remote_username: "{{ dev_pulp_username }}" - remote_password: "{{ dev_pulp_password }}" state: present # List of release container image repositories. @@ -23,8 +17,7 @@ dev_pulp_repository_container_repos_release: >- {%- for image in kolla_container_images_filtered -%} {%- if image not in kolla_unbuildable_images[base_distro]-%} {%- set image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%} - {%- set upstream_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%} - {%- set repo = {"name": image_repo, "upstream_name": upstream_repo} -%} + {%- set repo = {"name": image_repo} -%} {%- set _ = repos.append(dev_pulp_repository_container_repo_release_common | combine(repo)) -%} {%- endif -%} {%- endfor -%} diff --git a/ansible/roles/pulp-container-content/README.md b/ansible/roles/pulp-container-content/README.md new file mode 100644 index 00000000..9bdad408 --- /dev/null +++ b/ansible/roles/pulp-container-content/README.md @@ -0,0 +1,49 @@ +pulp_container_content +====================== + +This role adds and removes content in Pulp container repositories. + +Currently only supports tags. + +Role variables +-------------- + +* `pulp_url`: URL of Pulp server. Default is `https://localhost:8080` +* `pulp_username`: Username used to access Pulp server. Default is `admin` +* `pulp_password`: Password used to access Pulp server. Default is unset +* `pulp_container_content`: List of content to add or remove. Each item is a dict with the following keys: + + * `src_repo`: Name of the repository to copy from when `state` is `present`, + or the repository to remove from when `state` is `absent`. + * `src_is_push`: Whether `src_repo` is a push repository. Default is `false`. + * `dest_repo`: Name of the repository to copy to when `state is `present`. + * `tags`: List of names of tags to add or remove. + * `state`: Whether to add (`present`) or remove (`absent`) content. + +Example playbook +---------------- + +```yaml +--- +- name: Add or remove container content + any_errors_fatal: True + gather_facts: True + hosts: all + roles: + - role: pulp-container-content + pulp_username: admin + pulp_password: "{{ secrets_pulp_admin_password }}" + pulp_container_content: + # Copy tag1 and tag2 from repo1 to repo2 + - src_repo: repo1 + src_is_push: true + dest_repo: repo2 + tags: + - tag1 + - tag2 + # Remove tag3 from repo3 + - src_repo: repo3 + tags: + - tag3 + state: absent +``` diff --git a/ansible/roles/pulp-container-content/defaults/main.yml b/ansible/roles/pulp-container-content/defaults/main.yml new file mode 100644 index 00000000..e3310584 --- /dev/null +++ b/ansible/roles/pulp-container-content/defaults/main.yml @@ -0,0 +1,6 @@ +--- +pulp_url: https://localhost:8080 +pulp_username: admin +pulp_password: + +pulp_container_content: [] diff --git a/ansible/roles/pulp-container-content/tasks/content.yml b/ansible/roles/pulp-container-content/tasks/content.yml new file mode 100644 index 00000000..9de01c5b --- /dev/null +++ b/ansible/roles/pulp-container-content/tasks/content.yml @@ -0,0 +1,73 @@ +--- +- name: Query source repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container{% if item.src_is_push | default(false) | bool %}-push{% endif %}/?name={{ item.src_repo | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: src_repo + when: item.state | default != 'absent' + +- name: Assert that source repository exists + assert: + that: + src_repo.json.results | length > 0 + when: item.state | default != 'absent' + +- name: Query destination repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ item.dest_repo | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: dest_repo + +- name: Assert that destination repository exists + assert: + that: + dest_repo.json.results | length > 0 + +- name: Query tags + uri: + url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?name__in={{ item.tags | join(',') | urlencode | regex_replace('/','%2F') }}&repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: tags + vars: + repo: "{{ src_repo if item.state | default != 'absent' else dest_repo }}" + +# NOTE: Currently we don't verify that all tags exist. This can be fixed when we specify a different tag for each repo. + +- name: Add or remove content units + uri: + url: "{{ pulp_url }}{{ dest_repo.json.results[0].pulp_href }}{% if item.state | default != 'absent' %}add{% else %}remove{% endif %}/" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: POST + status_code: 202 + force_basic_auth: true + body: + content_units: "{{ tags.json.results | map(attribute='pulp_href') | list }}" + body_format: json + register: task + +- name: Wait for task to complete + uri: + url: "{{ pulp_url }}{{ task.json.task }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: result + until: result.json.state not in ['waiting', 'running'] + retries: 30 + delay: 2 + failed_when: result.json.state != 'completed' diff --git a/ansible/roles/pulp-container-content/tasks/main.yml b/ansible/roles/pulp-container-content/tasks/main.yml new file mode 100644 index 00000000..00e020b7 --- /dev/null +++ b/ansible/roles/pulp-container-content/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Include content.yml + include_tasks: content.yml + loop: "{{ pulp_container_content }}" From 209096e2beba8e84137a0c99d1537f2dd475389c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 16:35:31 +0100 Subject: [PATCH 2/2] Switch to stackhpc.pulp.pulp_container_content This is available in stackhpc.pulp collection v0.5.0, and is implemented in Python for a speed up. --- ansible/dev-pulp-container-promote.yml | 10 ++- .../roles/pulp-container-content/README.md | 49 ------------- .../pulp-container-content/defaults/main.yml | 6 -- .../pulp-container-content/tasks/content.yml | 73 ------------------- .../pulp-container-content/tasks/main.yml | 4 - requirements.yml | 4 +- 6 files changed, 10 insertions(+), 136 deletions(-) delete mode 100644 ansible/roles/pulp-container-content/README.md delete mode 100644 ansible/roles/pulp-container-content/defaults/main.yml delete mode 100644 ansible/roles/pulp-container-content/tasks/content.yml delete mode 100644 ansible/roles/pulp-container-content/tasks/main.yml diff --git a/ansible/dev-pulp-container-promote.yml b/ansible/dev-pulp-container-promote.yml index e2cba475..4cb486aa 100644 --- a/ansible/dev-pulp-container-promote.yml +++ b/ansible/dev-pulp-container-promote.yml @@ -36,7 +36,7 @@ # Copy tags from stackhpc-dev to stackhpc repositories. - import_role: - name: pulp-container-content + name: stackhpc.pulp.pulp_container_content vars: pulp_url: "{{ dev_pulp_url }}" pulp_username: "{{ dev_pulp_username }}" @@ -48,7 +48,13 @@ {%- if image not in kolla_unbuildable_images[base_distro]-%} {%- set src_image_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%} {%- set dest_image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%} - {%- set content = {"src_repo": src_image_repo, "src_is_push": true, "dest_repo": dest_image_repo, "tags": [dev_pulp_repository_container_promotion_tag]} -%} + {%- set content = { + "allow_missing": True, + "src_repo": src_image_repo, + "src_is_push": true, + "repository": dest_image_repo, + "tags": [dev_pulp_repository_container_promotion_tag], + } -%} {%- set _ = contents.append(content) -%} {%- endif -%} {%- endfor -%} diff --git a/ansible/roles/pulp-container-content/README.md b/ansible/roles/pulp-container-content/README.md deleted file mode 100644 index 9bdad408..00000000 --- a/ansible/roles/pulp-container-content/README.md +++ /dev/null @@ -1,49 +0,0 @@ -pulp_container_content -====================== - -This role adds and removes content in Pulp container repositories. - -Currently only supports tags. - -Role variables --------------- - -* `pulp_url`: URL of Pulp server. Default is `https://localhost:8080` -* `pulp_username`: Username used to access Pulp server. Default is `admin` -* `pulp_password`: Password used to access Pulp server. Default is unset -* `pulp_container_content`: List of content to add or remove. Each item is a dict with the following keys: - - * `src_repo`: Name of the repository to copy from when `state` is `present`, - or the repository to remove from when `state` is `absent`. - * `src_is_push`: Whether `src_repo` is a push repository. Default is `false`. - * `dest_repo`: Name of the repository to copy to when `state is `present`. - * `tags`: List of names of tags to add or remove. - * `state`: Whether to add (`present`) or remove (`absent`) content. - -Example playbook ----------------- - -```yaml ---- -- name: Add or remove container content - any_errors_fatal: True - gather_facts: True - hosts: all - roles: - - role: pulp-container-content - pulp_username: admin - pulp_password: "{{ secrets_pulp_admin_password }}" - pulp_container_content: - # Copy tag1 and tag2 from repo1 to repo2 - - src_repo: repo1 - src_is_push: true - dest_repo: repo2 - tags: - - tag1 - - tag2 - # Remove tag3 from repo3 - - src_repo: repo3 - tags: - - tag3 - state: absent -``` diff --git a/ansible/roles/pulp-container-content/defaults/main.yml b/ansible/roles/pulp-container-content/defaults/main.yml deleted file mode 100644 index e3310584..00000000 --- a/ansible/roles/pulp-container-content/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -pulp_url: https://localhost:8080 -pulp_username: admin -pulp_password: - -pulp_container_content: [] diff --git a/ansible/roles/pulp-container-content/tasks/content.yml b/ansible/roles/pulp-container-content/tasks/content.yml deleted file mode 100644 index 9de01c5b..00000000 --- a/ansible/roles/pulp-container-content/tasks/content.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -- name: Query source repository - uri: - url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container{% if item.src_is_push | default(false) | bool %}-push{% endif %}/?name={{ item.src_repo | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: src_repo - when: item.state | default != 'absent' - -- name: Assert that source repository exists - assert: - that: - src_repo.json.results | length > 0 - when: item.state | default != 'absent' - -- name: Query destination repository - uri: - url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ item.dest_repo | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: dest_repo - -- name: Assert that destination repository exists - assert: - that: - dest_repo.json.results | length > 0 - -- name: Query tags - uri: - url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?name__in={{ item.tags | join(',') | urlencode | regex_replace('/','%2F') }}&repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: tags - vars: - repo: "{{ src_repo if item.state | default != 'absent' else dest_repo }}" - -# NOTE: Currently we don't verify that all tags exist. This can be fixed when we specify a different tag for each repo. - -- name: Add or remove content units - uri: - url: "{{ pulp_url }}{{ dest_repo.json.results[0].pulp_href }}{% if item.state | default != 'absent' %}add{% else %}remove{% endif %}/" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: POST - status_code: 202 - force_basic_auth: true - body: - content_units: "{{ tags.json.results | map(attribute='pulp_href') | list }}" - body_format: json - register: task - -- name: Wait for task to complete - uri: - url: "{{ pulp_url }}{{ task.json.task }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: result - until: result.json.state not in ['waiting', 'running'] - retries: 30 - delay: 2 - failed_when: result.json.state != 'completed' diff --git a/ansible/roles/pulp-container-content/tasks/main.yml b/ansible/roles/pulp-container-content/tasks/main.yml deleted file mode 100644 index 00e020b7..00000000 --- a/ansible/roles/pulp-container-content/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: Include content.yml - include_tasks: content.yml - loop: "{{ pulp_container_content }}" diff --git a/requirements.yml b/requirements.yml index 73130d71..b78e33a1 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,7 +1,7 @@ collections: - name: stackhpc.pulp - version: 0.4.1 + version: 0.5.0 - name: community.crypto version: 2.0.2 - name: pulp.squeezer - version: 0.0.13 \ No newline at end of file + version: 0.0.13