From c64afbd0dd42337b5508c11e47a406b36e076623 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 16:54:18 +0100 Subject: [PATCH 1/5] Add pulp_container_content role This role uses the "Add content" API to add or remove container tags to/from a repository. [1] https://docs.pulpproject.org/pulp_container/restapi.html#tag/Repositories:-Container/operation/repositories_container_container_add --- roles/pulp_container_content/README.md | 49 +++++++++++++ .../pulp_container_content/defaults/main.yml | 6 ++ .../pulp_container_content/tasks/content.yml | 73 +++++++++++++++++++ roles/pulp_container_content/tasks/main.yml | 4 + 4 files changed, 132 insertions(+) create mode 100644 roles/pulp_container_content/README.md create mode 100644 roles/pulp_container_content/defaults/main.yml create mode 100644 roles/pulp_container_content/tasks/content.yml create mode 100644 roles/pulp_container_content/tasks/main.yml diff --git a/roles/pulp_container_content/README.md b/roles/pulp_container_content/README.md new file mode 100644 index 0000000..6934492 --- /dev/null +++ b/roles/pulp_container_content/README.md @@ -0,0 +1,49 @@ +pulp_container_content +====================== + +This role adds and removes content in Pulp container repositories. + +Currently only supports tags. + +Role variables +-------------- + +* `pulp_url`: URL of Pulp server. Default is `https://localhost:8080` +* `pulp_username`: Username used to access Pulp server. Default is `admin` +* `pulp_password`: Password used to access Pulp server. Default is unset +* `pulp_container_content`: List of content to add or remove. Each item is a dict with the following keys: + + * `src_repo`: Name of the repository to copy from when `state` is `present`, + or the repository to remove from when `state` is `absent`. + * `src_is_push`: Whether `src_repo` is a push repository. Default is `false`. + * `dest_repo`: Name of the repository to copy to when `state is `present`. + * `tags`: List of names of tags to add or remove. + * `state`: Whether to add (`present`) or remove (`absent`) content. + +Example playbook +---------------- + +```yaml +--- +- name: Add or remove container content + any_errors_fatal: True + gather_facts: True + hosts: all + roles: + - role: pulp_container_content + pulp_username: admin + pulp_password: "{{ secrets_pulp_admin_password }}" + pulp_container_content: + # Copy tag1 and tag2 from repo1 to repo2 + - src_repo: repo1 + src_is_push: true + dest_repo: repo2 + tags: + - tag1 + - tag2 + # Remove tag3 from repo3 + - src_repo: repo3 + tags: + - tag3 + state: absent +``` diff --git a/roles/pulp_container_content/defaults/main.yml b/roles/pulp_container_content/defaults/main.yml new file mode 100644 index 0000000..e331058 --- /dev/null +++ b/roles/pulp_container_content/defaults/main.yml @@ -0,0 +1,6 @@ +--- +pulp_url: https://localhost:8080 +pulp_username: admin +pulp_password: + +pulp_container_content: [] diff --git a/roles/pulp_container_content/tasks/content.yml b/roles/pulp_container_content/tasks/content.yml new file mode 100644 index 0000000..9de01c5 --- /dev/null +++ b/roles/pulp_container_content/tasks/content.yml @@ -0,0 +1,73 @@ +--- +- name: Query source repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container{% if item.src_is_push | default(false) | bool %}-push{% endif %}/?name={{ item.src_repo | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: src_repo + when: item.state | default != 'absent' + +- name: Assert that source repository exists + assert: + that: + src_repo.json.results | length > 0 + when: item.state | default != 'absent' + +- name: Query destination repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ item.dest_repo | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: dest_repo + +- name: Assert that destination repository exists + assert: + that: + dest_repo.json.results | length > 0 + +- name: Query tags + uri: + url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?name__in={{ item.tags | join(',') | urlencode | regex_replace('/','%2F') }}&repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: tags + vars: + repo: "{{ src_repo if item.state | default != 'absent' else dest_repo }}" + +# NOTE: Currently we don't verify that all tags exist. This can be fixed when we specify a different tag for each repo. + +- name: Add or remove content units + uri: + url: "{{ pulp_url }}{{ dest_repo.json.results[0].pulp_href }}{% if item.state | default != 'absent' %}add{% else %}remove{% endif %}/" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: POST + status_code: 202 + force_basic_auth: true + body: + content_units: "{{ tags.json.results | map(attribute='pulp_href') | list }}" + body_format: json + register: task + +- name: Wait for task to complete + uri: + url: "{{ pulp_url }}{{ task.json.task }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: result + until: result.json.state not in ['waiting', 'running'] + retries: 30 + delay: 2 + failed_when: result.json.state != 'completed' diff --git a/roles/pulp_container_content/tasks/main.yml b/roles/pulp_container_content/tasks/main.yml new file mode 100644 index 0000000..00e020b --- /dev/null +++ b/roles/pulp_container_content/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Include content.yml + include_tasks: content.yml + loop: "{{ pulp_container_content }}" From bb265dd73a83e196996bf637be6b621ee4adc3e2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 16:57:05 +0100 Subject: [PATCH 2/5] Convert pulp-container-content into an Ansible module This should execute more quickly than Ansible tasks. --- plugins/modules/pulp_container_content.py | 189 ++++++++++++++++++ roles/pulp_container_content/README.md | 13 +- .../pulp_container_content/defaults/main.yml | 1 + .../pulp_container_content/tasks/content.yml | 73 ------- roles/pulp_container_content/tasks/main.yml | 14 +- 5 files changed, 210 insertions(+), 80 deletions(-) create mode 100644 plugins/modules/pulp_container_content.py delete mode 100644 roles/pulp_container_content/tasks/content.yml diff --git a/plugins/modules/pulp_container_content.py b/plugins/modules/pulp_container_content.py new file mode 100644 index 0000000..694416e --- /dev/null +++ b/plugins/modules/pulp_container_content.py @@ -0,0 +1,189 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + + +DOCUMENTATION = r""" +--- +module: pulp_container_content +short_description: Manage container content of a pulp api server instance +description: + - "This performs CRUD operations on container content in a pulp api server instance." +options: + allow_missing: + description: + - Whether to allow missing tags when state is present. + type: bool + default: false + src_repo: + description: + - Name of the repository to copy content from when state is present. + type: str + src_is_push: + description: + - Whether src_repo is a container-push repository. + type: bool + default: false + repository: + description: + - Name of the repository to add or remove content + type: str + required: true + tags: + description: + - List of tags to add or remove + type: list + items: str + required: true +extends_documentation_fragment: + - pulp.squeezer.pulp + - pulp.squeezer.pulp.entity_state +author: + - Mark Goddard (@markgoddard) +""" + +EXAMPLES = r""" +- name: Copy tag1 and tag2 from repo1 to repo2 + pulp_container_content: + pulp_url: https://pulp.example.org + username: admin + password: password + repository: repo2 + src_repo: repo1 + tags: + - tag1 + - tag2 + +- name: Remove tag3 from repo3 + pulp_container_content: + pulp_url: https://pulp.example.org + username: admin + password: password + repository: repo3 + tags: + - tag3 + state: absent +""" + +RETURN = r""" + repository_version: + description: Created container repository version + type: dict + returned: when content is added or removed +""" + + +from ansible_collections.pulp.squeezer.plugins.module_utils.pulp import ( + PAGE_LIMIT, + PulpContainerRepository, + PulpEntity, + PulpEntityAnsibleModule, + PulpTask, + SqueezerException, +) + + +class PulpContainerRepositoryContent(PulpContainerRepository): + _add_id = "repositories_container_container_add" + _remove_id = "repositories_container_container_remove" + _container_tags_list_id = "content_container_tags_list" + + _name_singular = "repository_version" + + def get_src_repo(self): + # Query source repository. + natural_key = {"name": self.module.params["src_repo"]} + repo = PulpContainerRepository(self.module, natural_key) + if self.module.params["state"] == "present" and self.module.params["src_is_push"]: + repo._list_id = "repositories_container_container_push_list" + # find populates repo.entity. + repo.find(failsafe=False) + return repo + + def get_content_units(self, repo): + # Query container tags with matching names in repo. + # Pagination code adapted from PulpEntity.list(). + tags = [] + offset = 0 + search_result = {"next": True} + while search_result["next"]: + parameters = { + "limit": PAGE_LIMIT, + "offset": offset, + "name__in": ",".join(self.module.params["tags"]), + "repository_version": repo.entity["latest_version_href"] + } + search_result = self.module.pulp_api.call( + self._container_tags_list_id, parameters=parameters + ) + tags.extend(search_result["results"]) + offset += PAGE_LIMIT + + if (self.module.params["state"] == "present" and + not self.module.params["allow_missing"] and + len(tags) != len(self.module.params["tags"])): + missing = ", ".join(set(self.module.params["tags"]) - set(tags)) + raise SqueezerException(f"Some tags not found in source repository: {missing}") + return [result["pulp_href"] for result in tags] + + def add_or_remove(self, add_or_remove_id, content_units): + body = {"content_units": content_units} + if not self.module.check_mode: + parameters = {"container_container_repository_href": self.entity["pulp_href"]} + response = self.module.pulp_api.call( + add_or_remove_id, body=body, uploads=self.uploads, parameters=parameters + ) + if response and "task" in response: + task = PulpTask(self.module, {"pulp_href": response["task"]}).wait_for() + # Adding or removing content results in creation of a new repository version + if task["created_resources"]: + self.entity = {"pulp_href": task["created_resources"][0]} + self.module.set_changed() + else: + self.entity = None + else: + self.entity = response + else: + # Assume changed in check mode + self.module.set_changed() + + def add(self): + src_repo = self.get_src_repo() + self.add_or_remove(self._add_id, self.get_content_units(src_repo)) + + def remove(self): + self.add_or_remove(self._remove_id, self.get_content_units(self)) + + def process(self): + # Populate self.entity. + self.find(failsafe=False) + if self.module.params["state"] == "present": + response = self.add() + elif self.module.params["state"] == "absent": + response = self.remove() + else: + raise SqueezerException("Unexpected state") + self.module.set_result(self._name_singular, self.presentation(self.entity)) + + +def main(): + with PulpEntityAnsibleModule( + argument_spec=dict( + allow_missing={"type": "bool", "default": False}, + repository={"required": True}, + src_repo={}, + src_is_push={"type": "bool", "default": False}, + state={"default": "present"}, + tags={"type": "list", "item": "str", "required": True}, + ), + required_if=[("state", "present", ["src_repo"])], + ) as module: + natural_key = {"name": module.params["repository"]} + PulpContainerRepositoryContent(module, natural_key).process() + + +if __name__ == "__main__": + main() diff --git a/roles/pulp_container_content/README.md b/roles/pulp_container_content/README.md index 6934492..028fe46 100644 --- a/roles/pulp_container_content/README.md +++ b/roles/pulp_container_content/README.md @@ -11,14 +11,17 @@ Role variables * `pulp_url`: URL of Pulp server. Default is `https://localhost:8080` * `pulp_username`: Username used to access Pulp server. Default is `admin` * `pulp_password`: Password used to access Pulp server. Default is unset +* `pulp_validate_certs`: Whether to validate certificates. Default is `true`. * `pulp_container_content`: List of content to add or remove. Each item is a dict with the following keys: - * `src_repo`: Name of the repository to copy from when `state` is `present`, + * `allow_missing`: Whether to ignore missing tags in the source repository + when `state` is `present`. + * `repository`: Name of the repository to copy to when `state is `present` or the repository to remove from when `state` is `absent`. + * `src_repo`: Name of the repository to copy from when `state` is `present`. * `src_is_push`: Whether `src_repo` is a push repository. Default is `false`. - * `dest_repo`: Name of the repository to copy to when `state is `present`. - * `tags`: List of names of tags to add or remove. * `state`: Whether to add (`present`) or remove (`absent`) content. + * `tags`: List of names of tags to add or remove. Example playbook ---------------- @@ -37,12 +40,12 @@ Example playbook # Copy tag1 and tag2 from repo1 to repo2 - src_repo: repo1 src_is_push: true - dest_repo: repo2 + repository: repo2 tags: - tag1 - tag2 # Remove tag3 from repo3 - - src_repo: repo3 + - repository: repo3 tags: - tag3 state: absent diff --git a/roles/pulp_container_content/defaults/main.yml b/roles/pulp_container_content/defaults/main.yml index e331058..bfcbd46 100644 --- a/roles/pulp_container_content/defaults/main.yml +++ b/roles/pulp_container_content/defaults/main.yml @@ -2,5 +2,6 @@ pulp_url: https://localhost:8080 pulp_username: admin pulp_password: +pulp_validate_certs: true pulp_container_content: [] diff --git a/roles/pulp_container_content/tasks/content.yml b/roles/pulp_container_content/tasks/content.yml deleted file mode 100644 index 9de01c5..0000000 --- a/roles/pulp_container_content/tasks/content.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -- name: Query source repository - uri: - url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container{% if item.src_is_push | default(false) | bool %}-push{% endif %}/?name={{ item.src_repo | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: src_repo - when: item.state | default != 'absent' - -- name: Assert that source repository exists - assert: - that: - src_repo.json.results | length > 0 - when: item.state | default != 'absent' - -- name: Query destination repository - uri: - url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ item.dest_repo | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: dest_repo - -- name: Assert that destination repository exists - assert: - that: - dest_repo.json.results | length > 0 - -- name: Query tags - uri: - url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?name__in={{ item.tags | join(',') | urlencode | regex_replace('/','%2F') }}&repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: tags - vars: - repo: "{{ src_repo if item.state | default != 'absent' else dest_repo }}" - -# NOTE: Currently we don't verify that all tags exist. This can be fixed when we specify a different tag for each repo. - -- name: Add or remove content units - uri: - url: "{{ pulp_url }}{{ dest_repo.json.results[0].pulp_href }}{% if item.state | default != 'absent' %}add{% else %}remove{% endif %}/" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: POST - status_code: 202 - force_basic_auth: true - body: - content_units: "{{ tags.json.results | map(attribute='pulp_href') | list }}" - body_format: json - register: task - -- name: Wait for task to complete - uri: - url: "{{ pulp_url }}{{ task.json.task }}" - user: "{{ pulp_username }}" - password: "{{ pulp_password }}" - method: GET - status_code: 200 - force_basic_auth: true - register: result - until: result.json.state not in ['waiting', 'running'] - retries: 30 - delay: 2 - failed_when: result.json.state != 'completed' diff --git a/roles/pulp_container_content/tasks/main.yml b/roles/pulp_container_content/tasks/main.yml index 00e020b..927dbd4 100644 --- a/roles/pulp_container_content/tasks/main.yml +++ b/roles/pulp_container_content/tasks/main.yml @@ -1,4 +1,14 @@ --- -- name: Include content.yml - include_tasks: content.yml +- name: Add or remove content units + stackhpc.pulp.pulp_container_content: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs | bool }}" + allow_missing: "{{ item.allow_missing | default(omit) }}" + src_repo: "{{ item.src_repo | default(omit) }}" + src_is_push: "{{ item.src_is_push | default(omit) }}" + repository: "{{ item.repository }}" + tags: "{{ item.tags }}" + state: "{{ item.state | default(omit) }}" loop: "{{ pulp_container_content }}" From addfeac7b57c037a396481d5f950ad63467c269b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 19:59:16 +0100 Subject: [PATCH 3/5] pulp_container_content: Add an integration test --- tests/test_container_content.yml | 197 +++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 tests/test_container_content.yml diff --git a/tests/test_container_content.yml b/tests/test_container_content.yml new file mode 100644 index 0000000..0b3ad5b --- /dev/null +++ b/tests/test_container_content.yml @@ -0,0 +1,197 @@ +--- +- name: Test container repositories + gather_facts: false + hosts: localhost + vars: + pulp_url: http://localhost:8080 + pulp_username: admin + pulp_password: password + pulp_validate_certs: true + tasks: + - include_role: + name: pulp_repository + vars: + pulp_repository_container_repos: + - name: test_container_repo + upstream_name: pulp/test-fixture-1 + url: "https://registry-1.docker.io" + policy: immediate + state: present + - name: test_container_repo2 + state: present + + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + src_repo: test_container_repo + tags: + - manifest_a + - manifest_b + state: present + + - name: Query repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ 'test_container_repo2' | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: repo + + - name: Query tags + uri: + url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: tags + + - name: Assert that tags have been added + assert: + that: + - tags.json.results | map(attribute='name') | sort | list == ['manifest_a', 'manifest_b'] + + # Test idempotence + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + src_repo: test_container_repo + tags: + - manifest_a + - manifest_b + state: present + + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + tags: + - manifest_b + state: absent + + - name: Query repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ 'test_container_repo2' | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: repo + + - name: Query tags + uri: + url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: tags + + - name: Assert that manifest_b tag has been removed + assert: + that: + - tags.json.results | map(attribute='name') | list == ['manifest_a'] + + # Test idempotence + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + tags: + - manifest_b + state: absent + + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + tags: + - manifest_a + state: absent + + - name: Query repository + uri: + url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ 'test_container_repo2' | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: repo + + - name: Query tags + uri: + url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F') }}" + user: "{{ pulp_username }}" + password: "{{ pulp_password }}" + method: GET + status_code: 200 + force_basic_auth: true + register: tags + + - name: Assert that all tags have been removed + assert: + that: + - tags.json.results == [] + + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + tags: + - manifest_a + state: absent + + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - allow_missing: true + repository: test_container_repo2 + src_repo: test_container_repo + tags: + - not-a-valid-tag + state: present + + # When allow_missing is false (this is the default), the role should fail + # when provided with a that is not in the source repository. + - block: + - include_role: + name: pulp_container_content + vars: + pulp_container_content: + - repository: test_container_repo2 + src_repo: test_container_repo + tags: + - not-a-valid-tag + state: present + rescue: + - set_fact: + failed_task: "{{ ansible_failed_task }}" + always: + - name: Assert that adding a missing tag failed + assert: + that: + - failed_task.name == "Add or remove content units" + + - include_role: + name: pulp_repository + vars: + pulp_repository_container_repos: + - name: test_container_repo + state: absent + - name: test_container_repo2 + state: absent From b3e9c7f0c00eb35f86d48a816dd43e93cdd14ee6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 20:04:46 +0100 Subject: [PATCH 4/5] CI: use color --- .github/workflows/pull_request.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 72afa4a..2dba20d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,6 +7,8 @@ on: push: branches: - master +env: + ANSIBLE_FORCE_COLOR: True jobs: lint: From 300d4c1edd909a7b63043f5ab95e957c63c4c2d2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 6 Apr 2023 20:49:37 +0100 Subject: [PATCH 5/5] Set version to 0.5.0 --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 6030a9f..a656ca6 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,6 +1,6 @@ namespace: "stackhpc" name: "pulp" -version: "0.4.2" +version: "0.5.0" readme: "README.md" authors: - "Piotr Parczewski"