From 1329e72b089ba8f91f3d8d470816754a9611414f Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 25 Apr 2023 11:09:08 +0100 Subject: [PATCH 1/2] Add retries to repository sync tasks Repository syncs can fail for many reasons that may be transient. Add configurable retries to sync tasks, by default retrying up to three times if there is a sync failure. --- roles/pulp_repository/defaults/main.yml | 7 +++++++ roles/pulp_repository/tasks/container.yml | 4 ++++ roles/pulp_repository/tasks/deb.yml | 4 ++++ roles/pulp_repository/tasks/python.yml | 5 +++++ roles/pulp_repository/tasks/rpm.yml | 4 ++++ 5 files changed, 24 insertions(+) diff --git a/roles/pulp_repository/defaults/main.yml b/roles/pulp_repository/defaults/main.yml index c3702bd..7f02d0d 100644 --- a/roles/pulp_repository/defaults/main.yml +++ b/roles/pulp_repository/defaults/main.yml @@ -8,3 +8,10 @@ pulp_repository_container_repos: [] pulp_repository_deb_repos: [] pulp_repository_python_repos: [] pulp_repository_rpm_repos: [] + +pulp_repository_sync_retries: 3 + +pulp_repository_container_repos_sync_retries: "{{ pulp_repository_sync_retries }}" +pulp_repository_deb_repos_sync_retries: "{{ pulp_repository_sync_retries }}" +pulp_repository_python_repos_sync_retries: "{{ pulp_repository_sync_retries }}" +pulp_repository_rpm_repos_sync_retries: "{{ pulp_repository_sync_retries }}" diff --git a/roles/pulp_repository/tasks/container.yml b/roles/pulp_repository/tasks/container.yml index 3a32cd4..58a6ce2 100644 --- a/roles/pulp_repository/tasks/container.yml +++ b/roles/pulp_repository/tasks/container.yml @@ -55,3 +55,7 @@ loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" loop_control: index_var: repository_index + register: pulp_repository_container_repos_sync + until: "pulp_repository_container_repos_sync is not failed" + retries: "{{ pulp_repository_container_repos_sync_retries }}" + delay: 1 diff --git a/roles/pulp_repository/tasks/deb.yml b/roles/pulp_repository/tasks/deb.yml index 39921f1..884202a 100644 --- a/roles/pulp_repository/tasks/deb.yml +++ b/roles/pulp_repository/tasks/deb.yml @@ -56,3 +56,7 @@ loop: "{{ pulp_repository_deb_repos | map(attribute='name') }}" loop_control: index_var: repository_index + register: pulp_repository_deb_repos_sync + until: "pulp_repository_deb_repos_sync is not failed" + retries: "{{ pulp_repository_deb_repos_sync_retries }}" + delay: 1 diff --git a/roles/pulp_repository/tasks/python.yml b/roles/pulp_repository/tasks/python.yml index b7e95e7..96a140b 100644 --- a/roles/pulp_repository/tasks/python.yml +++ b/roles/pulp_repository/tasks/python.yml @@ -55,3 +55,8 @@ loop: "{{ pulp_repository_python_repos | map(attribute='name') }}" loop_control: index_var: repository_index + register: pulp_repository_python_repos_sync + until: "pulp_repository_python_repos_sync is not failed" + retries: "{{ pulp_repository_python_repos_sync_retries }}" + delay: 1 + diff --git a/roles/pulp_repository/tasks/rpm.yml b/roles/pulp_repository/tasks/rpm.yml index 6388b20..b24253c 100644 --- a/roles/pulp_repository/tasks/rpm.yml +++ b/roles/pulp_repository/tasks/rpm.yml @@ -53,3 +53,7 @@ loop: "{{ pulp_repository_rpm_repos | map(attribute='name') }}" loop_control: index_var: repository_index + register: pulp_repository_rpm_repos_sync + until: "pulp_repository_rpm_repos_sync is not failed" + retries: "{{ pulp_repository_rpm_repos_sync_retries }}" + delay: 1 From 7dc4e616b78a4d39757c0e8b45905f35b637462b Mon Sep 17 00:00:00 2001 From: Matt Anson Date: Tue, 25 Apr 2023 11:59:35 +0100 Subject: [PATCH 2/2] Add tests --- tests/test_container_repository.yml | 34 +++++++++++++++++++++++++++++ tests/test_deb_repository.yml | 33 ++++++++++++++++++++++++++++ tests/test_rpm_repository.yml | 32 +++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/tests/test_container_repository.yml b/tests/test_container_repository.yml index c5379a3..52aa9d4 100644 --- a/tests/test_container_repository.yml +++ b/tests/test_container_repository.yml @@ -7,6 +7,7 @@ pulp_username: admin pulp_password: password pulp_validate_certs: true + pulp_repository_container_repos_sync_retries: 2 tasks: - include_role: name: pulp_repository @@ -79,3 +80,36 @@ - name: Verify remote deletion assert: that: container_remotes.remotes | length == 0 + + - block: + - include_role: + name: pulp_repository + vars: + pulp_repository_container_repos: + - name: test_container_repo_bad_url + upstream_name: pulp/test-fixture-1 + url: "https://google.com/404" + policy: immediate + state: present + + rescue: + - set_fact: + failed_task: "{{ ansible_failed_task }}" + + always: + - name: Assert that syncing from a URL that returns 404 fails + assert: + that: + - failed_task.name == "Sync container remotes into repositories" + + - name: Assert that syncing from a URL that returns 404 is retried the correct number of times + assert: + that: + - pulp_repository_container_repos_sync.results[0].attempts == pulp_repository_container_repos_sync_retries + + - include_role: + name: pulp_repository + vars: + pulp_repository_deb_repos: + - name: test_container_repo_bad_url + state: absent \ No newline at end of file diff --git a/tests/test_deb_repository.yml b/tests/test_deb_repository.yml index 57e6354..3894f93 100644 --- a/tests/test_deb_repository.yml +++ b/tests/test_deb_repository.yml @@ -7,6 +7,7 @@ pulp_username: admin pulp_password: password pulp_validate_certs: true + pulp_repository_deb_repos_sync_retries: 2 tasks: - include_role: name: pulp_repository @@ -79,3 +80,35 @@ - name: Verify remote deletion assert: that: deb_remotes.remotes | length == 0 + + - block: + - include_role: + name: pulp_repository + vars: + pulp_repository_deb_repos: + - name: test_deb_repo_bad_url + url: "https://google.com/404" + distributions: "ragnarok" + policy: immediate + state: present + rescue: + - set_fact: + failed_task: "{{ ansible_failed_task }}" + + always: + - name: Assert that syncing from a URL that returns 404 fails + assert: + that: + - failed_task.name == "Sync DEB remotes into repositories" + + - name: Assert that syncing from a URL that returns 404 is retried the correct number of times + assert: + that: + - pulp_repository_deb_repos_sync.results[0].attempts == pulp_repository_deb_repos_sync_retries + + - include_role: + name: pulp_repository + vars: + pulp_repository_deb_repos: + - name: test_deb_repo_bad_url + state: absent \ No newline at end of file diff --git a/tests/test_rpm_repository.yml b/tests/test_rpm_repository.yml index b19b861..d2b2d78 100644 --- a/tests/test_rpm_repository.yml +++ b/tests/test_rpm_repository.yml @@ -7,6 +7,7 @@ pulp_username: admin pulp_password: password pulp_validate_certs: true + pulp_repository_rpm_repos_sync_retries: 2 tasks: - include_role: name: pulp_repository @@ -77,3 +78,34 @@ - name: Verify remote deletion assert: that: rpm_remotes.remotes | length == 0 + + - block: + - include_role: + name: pulp_repository + vars: + pulp_repository_rpm_repos: + - name: test_rpm_repo_bad_url + url: "https://google.com/404" + policy: immediate + state: present + rescue: + - set_fact: + failed_task: "{{ ansible_failed_task }}" + + always: + - name: Assert that syncing from a URL that returns 404 fails + assert: + that: + - failed_task.name == "Sync RPM remotes into repositories" + + - name: Assert that syncing from a URL that returns 404 is retried the correct number of times + assert: + that: + - pulp_repository_rpm_repos_sync.results[0].attempts == pulp_repository_rpm_repos_sync_retries + + - include_role: + name: pulp_repository + vars: + pulp_repository_rpm_repos: + - name: test_rpm_repo_bad_url + state: absent \ No newline at end of file