diff --git a/.github/workflows/package-promote.yml b/.github/workflows/package-promote.yml index eb4d1e43..a07db1dd 100644 --- a/.github/workflows/package-promote.yml +++ b/.github/workflows/package-promote.yml @@ -3,6 +3,11 @@ name: Promote package repositories on: workflow_dispatch: inputs: + filter: + description: Space-separated list of regular expressions matching short_name of repositories to promote + type: string + required: false + default: "" kayobe_config_branch: required: false description: Branch of StackHPC Kayobe configuration to use @@ -47,4 +52,8 @@ jobs: ansible-playbook -i ansible/inventory \ ansible/dev-pulp-repo-version-query-kayobe.yml \ ansible/dev-pulp-repo-promote.yml \ + -e deb_package_repo_filter="'$FILTER'" \ + -e rpm_package_repo_filter="'$FILTER'" \ -e kayobe_config_repo_path=./stackhpc-kayobe-config/ + env: + FILTER: ${{ github.event.inputs.filter }} diff --git a/.github/workflows/package-sync.yml b/.github/workflows/package-sync.yml index ed5ed6f6..8f3cf07d 100644 --- a/.github/workflows/package-sync.yml +++ b/.github/workflows/package-sync.yml @@ -13,6 +13,11 @@ on: description: Sync package repositories in Test Pulp default: true type: boolean + filter: + description: Space-separated list of regular expressions matching short_name of repositories to sync + type: string + required: false + default: "" schedule: # Daily at 02:17 - cron: '17 2 * * *' @@ -50,7 +55,11 @@ jobs: ansible-playbook -i ansible/inventory \ ansible/dev-pulp-repo-sync.yml \ ansible/dev-pulp-repo-publication-cleanup.yml \ - ansible/dev-pulp-repo-publish.yml + ansible/dev-pulp-repo-publish.yml \ + -e deb_package_repo_filter="'$FILTER'" \ + -e rpm_package_repo_filter="'$FILTER'" + env: + FILTER: ${{ github.event.inputs.filter }} package-sync-test: name: Sync package repositories in test @@ -82,4 +91,8 @@ jobs: ansible-playbook -i ansible/inventory \ ansible/test-pulp-repo-version-query.yml \ ansible/test-pulp-repo-sync.yml \ - ansible/test-pulp-repo-publish.yml + ansible/test-pulp-repo-publish.yml \ + -e deb_package_repo_filter="'$FILTER'" \ + -e rpm_package_repo_filter="'$FILTER'" + env: + FILTER: ${{ github.event.inputs.filter }} diff --git a/.github/workflows/package-update-kayobe.yml b/.github/workflows/package-update-kayobe.yml index 8fd1e1fb..5329cdb1 100644 --- a/.github/workflows/package-update-kayobe.yml +++ b/.github/workflows/package-update-kayobe.yml @@ -3,6 +3,11 @@ name: Update Kayobe package repository versions on: workflow_dispatch: inputs: + filter: + description: Space-separated list of regular expressions matching short_name of repositories to update + type: string + required: false + default: "" kayobe_config_branch: required: false description: Branch of StackHPC Kayobe configuration to use @@ -47,7 +52,11 @@ jobs: ansible-playbook -i ansible/inventory \ ansible/test-pulp-repo-version-query.yml \ ansible/test-kayobe-repo-version-generate.yml \ + -e deb_package_repo_filter="'$FILTER'" \ + -e rpm_package_repo_filter="'$FILTER'" \ -e kayobe_config_repo_path=./stackhpc-kayobe-config/ + env: + FILTER: ${{ github.event.inputs.filter }} - name: Check for version changes id: git-diff diff --git a/ansible/filter_plugins/filters.py b/ansible/filter_plugins/filters.py index c011a906..1a3c3024 100644 --- a/ansible/filter_plugins/filters.py +++ b/ansible/filter_plugins/filters.py @@ -15,6 +15,20 @@ import re +def select_repos(repos, filter_string): + """Select repositories that match a filter string. + + The filter string is a whitespace-separated list of regular expressions + matching repository short names. + """ + if not filter_string: + return repos + regexes = filter_string.split() + patterns = re.compile(r"|".join(regexes).join('()')) + return [repo for repo in repos + if "short_name" in repo and re.search(patterns, repo["short_name"])] + + def select_images(images, filter_string): """Select images that match a filter string. @@ -32,5 +46,6 @@ class FilterModule(object): def filters(self): return { + "select_repos": select_repos, "select_images": select_images, } diff --git a/ansible/inventory/group_vars/all/dev-pulp-repos b/ansible/inventory/group_vars/all/dev-pulp-repos index fe9f0be4..99b0c106 100644 --- a/ansible/inventory/group_vars/all/dev-pulp-repos +++ b/ansible/inventory/group_vars/all/dev-pulp-repos @@ -23,7 +23,7 @@ dev_pulp_repository_deb_repo_common: dev_pulp_repository_deb_repos: >- {%- set dev_repos = [] -%} - {%- for repo in deb_package_repos -%} + {%- for repo in deb_package_repos_filtered -%} {%- set dev_repo = {"name": repo.name} -%} {%- if repo.sync | default(true) -%} {%- set dev_repo = dev_repo | combine({"url": repo.url}) -%} @@ -57,7 +57,7 @@ dev_pulp_distribution_deb_common: dev_pulp_distribution_deb: >- {%- set dev_dists = [] -%} - {%- for repo in deb_package_repos -%} + {%- for repo in deb_package_repos_filtered -%} {%- if repo.publish | default(true) -%} {%- set version = dev_pulp_distribution_version -%} {%- set name = repo.distribution_name ~ version -%} @@ -83,7 +83,7 @@ dev_pulp_distribution_deb_promote_common: dev_pulp_distribution_deb_promote: >- {%- set dev_dists = [] -%} - {%- for repo in deb_package_repos -%} + {%- for repo in deb_package_repos_filtered -%} {%- if repo.publish | default(true) -%} {%- set version = dev_pulp_distribution_deb_promote_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version -%} @@ -107,7 +107,7 @@ dev_pulp_repository_rpm_repo_common: dev_pulp_repository_rpm_repos: >- {%- set dev_repos = [] -%} - {%- for repo in rpm_package_repos -%} + {%- for repo in rpm_package_repos_filtered -%} {%- set dev_repo = {"name": repo.name} -%} {%- if repo.sync | default(true) -%} {%- set dev_repo = dev_repo | combine({"url": repo.url}) -%} @@ -132,7 +132,7 @@ dev_pulp_distribution_rpm_common: dev_pulp_distribution_rpm: >- {%- set dev_dists = [] -%} - {%- for repo in rpm_package_repos -%} + {%- for repo in rpm_package_repos_filtered -%} {%- if repo.publish | default(true) -%} {%- set version = dev_pulp_distribution_version -%} {%- set name = repo.distribution_name ~ version -%} @@ -155,7 +155,7 @@ dev_pulp_distribution_rpm_promote_common: dev_pulp_distribution_rpm_promote: >- {%- set dev_dists = [] -%} - {%- for repo in rpm_package_repos -%} + {%- for repo in rpm_package_repos_filtered -%} {%- if repo.publish | default(true) -%} {%- set version = dev_pulp_distribution_rpm_promote_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version -%} diff --git a/ansible/inventory/group_vars/all/package-repos b/ansible/inventory/group_vars/all/package-repos index 2c0260e9..c7e698ea 100644 --- a/ansible/inventory/group_vars/all/package-repos +++ b/ansible/inventory/group_vars/all/package-repos @@ -79,6 +79,12 @@ deb_package_repos: short_name: docker_ce_ubuntu distribution_name: docker-ce-for-ubuntu- +# Default filter string for Deb package repositories. +deb_package_repo_filter: "" + +# List of package repositories after applying filter. +deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter) }}" + ############################################################################### # RPM @@ -345,3 +351,9 @@ rpm_package_repos: base_path: mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/ short_name: mlnx_ofed_5_6_1_0_3_3_rhel8_6 distribution_name: mlnx_ofed-5.6-1.0.3.3-rhel8.6- + +# Default filter string for RPM package repositories. +rpm_package_repo_filter: "" + +# List of package repositories after applying filter. +rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter) }}" diff --git a/ansible/inventory/group_vars/all/test-pulp-repos b/ansible/inventory/group_vars/all/test-pulp-repos index ff5bf9e5..06cb1c71 100644 --- a/ansible/inventory/group_vars/all/test-pulp-repos +++ b/ansible/inventory/group_vars/all/test-pulp-repos @@ -28,7 +28,7 @@ test_pulp_repository_deb_repo_versions: {} test_pulp_repository_deb_repos: >- {%- set test_repos = [] -%} - {%- for repo in deb_package_repos -%} + {%- for repo in deb_package_repos_filtered -%} {%- if repo.sync | default(true) and repo.publish | default(true) -%} {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%} {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%} @@ -55,7 +55,7 @@ test_pulp_distribution_deb_common: test_pulp_distribution_deb: >- {%- set test_dists = [] -%} - {%- for repo in deb_package_repos -%} + {%- for repo in deb_package_repos_filtered -%} {%- if repo.sync | default(true) and repo.publish | default(true) -%} {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version ~ "-ark" -%} @@ -92,7 +92,7 @@ test_pulp_repository_rpm_repo_versions: {} test_pulp_repository_rpm_repos: >- {%- set test_repos = [] -%} - {%- for repo in rpm_package_repos -%} + {%- for repo in rpm_package_repos_filtered -%} {%- if repo.sync | default(true) and repo.publish | default(true) -%} {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%} {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%} @@ -110,7 +110,7 @@ test_pulp_distribution_rpm_common: test_pulp_distribution_rpm: >- {%- set test_dists = [] -%} - {%- for repo in rpm_package_repos -%} + {%- for repo in rpm_package_repos_filtered -%} {%- if repo.sync | default(true) and repo.publish | default(true) -%} {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%} {%- set name = repo.distribution_name ~ version ~ "-ark" -%} diff --git a/ansible/test-kayobe-repo-version-generate.yml b/ansible/test-kayobe-repo-version-generate.yml index 455097a0..21202858 100644 --- a/ansible/test-kayobe-repo-version-generate.yml +++ b/ansible/test-kayobe-repo-version-generate.yml @@ -17,15 +17,6 @@ kayobe_pulp_repo_versions_path: "{{ (kayobe_config_repo_path ~ '/etc/kayobe/pulp-repo-versions.yml') | realpath }}" kayobe_pulp_repo_versions_dest: "{{ kayobe_pulp_repo_versions_path }}" tasks: - - name: Assert that versions variable is populated - assert: - that: - - missing_repos | length == 0 - msg: >- - Some expected repositories not present in Kayobe Pulp repo versions: {{ missing_repos | join(',') }} - vars: - missing_repos: "{{ kayobe_pulp_repo_versions.keys() | list | difference(test_pulp_repository_rpm_repo_versions) | difference(test_pulp_repository_deb_repo_versions) | list }}" - - name: Set a fact about updated Kayobe Pulp repository versions vars: kayobe_var_name: "{{ 'stackhpc_pulp_repo_' ~ item.key ~ '_version' }}" diff --git a/ansible/validate-deb-repos.yml b/ansible/validate-deb-repos.yml index 0ac6cf40..e144cfa4 100644 --- a/ansible/validate-deb-repos.yml +++ b/ansible/validate-deb-repos.yml @@ -53,6 +53,23 @@ that: - published_deb_package_repos | map(attribute='base_path') | list | map('last') | unique == ['/'] + - name: Assert that Deb package repository list can be filtered to single + assert: + that: + - deb_package_repos_filtered | length == 1 + - deb_package_repos_filtered[0].short_name == 'ubuntu_focal' + vars: + deb_package_repo_filter: ubuntu_focal$ + + - name: Assert that Deb package repository list can be filtered to multiple + assert: + that: + - deb_package_repos_filtered | length == 2 + - deb_package_repos_filtered[0].short_name == 'ubuntu_focal' + - deb_package_repos_filtered[1].short_name == 'docker_ce_ubuntu' + vars: + deb_package_repo_filter: docker_ce_ubuntu ubuntu_focal$ + - name: Assert that dev package repository list is defined assert: that: diff --git a/ansible/validate-rpm-repos.yml b/ansible/validate-rpm-repos.yml index 5c057f48..e1f499bb 100644 --- a/ansible/validate-rpm-repos.yml +++ b/ansible/validate-rpm-repos.yml @@ -53,6 +53,23 @@ that: - published_rpm_package_repos | map(attribute='base_path') | list | map('last') | unique == ['/'] + - name: Assert that RPM package repository list can be filtered to single + assert: + that: + - rpm_package_repos_filtered | length == 1 + - rpm_package_repos_filtered[0].short_name == 'centos_stream_8_baseos' + vars: + rpm_package_repo_filter: centos_stream_8_baseos + + - name: Assert that RPM package repository list can be filtered to multiple + assert: + that: + - rpm_package_repos_filtered | length == 2 + - rpm_package_repos_filtered[0].short_name == 'centos_stream_8_baseos' + - rpm_package_repos_filtered[1].short_name == 'docker' + vars: + rpm_package_repo_filter: docker centos_stream_8_baseos + - name: Assert that dev package repository list is defined assert: that: