Support filtering of package repositories

.github/workflows/package-promote.yml

@@ -3,6 +3,11 @@ name: Promote package repositories
 on:
   workflow_dispatch:
     inputs:
+      filter:
+        description: Space-separated list of regular expressions matching short_name of repositories to promote
+        type: string
+        required: false
+        default: ""
       kayobe_config_branch:
         required: false
         description: Branch of StackHPC Kayobe configuration to use
@@ -47,4 +52,8 @@ jobs:
         ansible-playbook -i ansible/inventory \
         ansible/dev-pulp-repo-version-query-kayobe.yml \
         ansible/dev-pulp-repo-promote.yml \
+        -e deb_package_repo_filter="'$FILTER'" \
+        -e rpm_package_repo_filter="'$FILTER'" \
         -e kayobe_config_repo_path=./stackhpc-kayobe-config/
+      env:
+        FILTER: ${{ github.event.inputs.filter }}

.github/workflows/package-sync.yml

@@ -13,6 +13,11 @@ on:
         description: Sync package repositories in Test Pulp
         default: true
         type: boolean
+      filter:
+        description: Space-separated list of regular expressions matching short_name of repositories to sync
+        type: string
+        required: false
+        default: ""
   schedule:
     # Daily at 02:17
     - cron: '17 2 * * *'
@@ -50,7 +55,11 @@ jobs:
         ansible-playbook -i ansible/inventory \
         ansible/dev-pulp-repo-sync.yml \
         ansible/dev-pulp-repo-publication-cleanup.yml \
-        ansible/dev-pulp-repo-publish.yml
+        ansible/dev-pulp-repo-publish.yml \
+        -e deb_package_repo_filter="'$FILTER'" \
+        -e rpm_package_repo_filter="'$FILTER'"
+      env:
+        FILTER: ${{ github.event.inputs.filter }}
 
   package-sync-test:
     name: Sync package repositories in test
@@ -82,4 +91,8 @@ jobs:
         ansible-playbook -i ansible/inventory \
         ansible/test-pulp-repo-version-query.yml \
         ansible/test-pulp-repo-sync.yml \
-        ansible/test-pulp-repo-publish.yml
+        ansible/test-pulp-repo-publish.yml \
+        -e deb_package_repo_filter="'$FILTER'" \
+        -e rpm_package_repo_filter="'$FILTER'"
+      env:
+        FILTER: ${{ github.event.inputs.filter }}

.github/workflows/package-update-kayobe.yml

@@ -3,6 +3,11 @@ name: Update Kayobe package repository versions
 on:
   workflow_dispatch:
     inputs:
+      filter:
+        description: Space-separated list of regular expressions matching short_name of repositories to update
+        type: string
+        required: false
+        default: ""
       kayobe_config_branch:
         required: false
         description: Branch of StackHPC Kayobe configuration to use
@@ -47,7 +52,11 @@ jobs:
           ansible-playbook -i ansible/inventory \
           ansible/test-pulp-repo-version-query.yml \
           ansible/test-kayobe-repo-version-generate.yml \
+          -e deb_package_repo_filter="'$FILTER'" \
+          -e rpm_package_repo_filter="'$FILTER'" \
           -e kayobe_config_repo_path=./stackhpc-kayobe-config/
+        env:
+          FILTER: ${{ github.event.inputs.filter }}
 
       - name: Check for version changes
         id: git-diff

ansible/filter_plugins/filters.py

@@ -15,6 +15,20 @@
 import re
 
 
+def select_repos(repos, filter_string):
+    """Select repositories that match a filter string.
+
+    The filter string is a whitespace-separated list of regular expressions
+    matching repository short names.
+    """
+    if not filter_string:
+        return repos
+    regexes = filter_string.split()
+    patterns = re.compile(r"|".join(regexes).join('()'))
+    return [repo for repo in repos
+            if "short_name" in repo and re.search(patterns, repo["short_name"])]
+
+
 def select_images(images, filter_string):
     """Select images that match a filter string.
 
@@ -32,5 +46,6 @@ class FilterModule(object):
 
     def filters(self):
         return {
+            "select_repos": select_repos,
             "select_images": select_images,
         }

ansible/inventory/group_vars/all/dev-pulp-repos

@@ -23,7 +23,7 @@ dev_pulp_repository_deb_repo_common:
 
 dev_pulp_repository_deb_repos: >-
   {%- set dev_repos = [] -%}
-  {%- for repo in deb_package_repos -%}
+  {%- for repo in deb_package_repos_filtered -%}
   {%- set dev_repo = {"name": repo.name} -%}
   {%- if repo.sync | default(true) -%}
   {%- set dev_repo = dev_repo | combine({"url": repo.url}) -%}
@@ -57,7 +57,7 @@ dev_pulp_distribution_deb_common:
 
 dev_pulp_distribution_deb: >-
   {%- set dev_dists = [] -%}
-  {%- for repo in deb_package_repos -%}
+  {%- for repo in deb_package_repos_filtered -%}
   {%- if repo.publish | default(true) -%}
   {%- set version = dev_pulp_distribution_version -%}
   {%- set name = repo.distribution_name ~ version -%}
@@ -83,7 +83,7 @@ dev_pulp_distribution_deb_promote_common:
 
 dev_pulp_distribution_deb_promote: >-
   {%- set dev_dists = [] -%}
-  {%- for repo in deb_package_repos -%}
+  {%- for repo in deb_package_repos_filtered -%}
   {%- if repo.publish | default(true) -%}
   {%- set version = dev_pulp_distribution_deb_promote_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version -%}
@@ -107,7 +107,7 @@ dev_pulp_repository_rpm_repo_common:
 
 dev_pulp_repository_rpm_repos: >-
   {%- set dev_repos = [] -%}
-  {%- for repo in rpm_package_repos -%}
+  {%- for repo in rpm_package_repos_filtered -%}
   {%- set dev_repo = {"name": repo.name} -%}
   {%- if repo.sync | default(true) -%}
   {%- set dev_repo = dev_repo | combine({"url": repo.url}) -%}
@@ -132,7 +132,7 @@ dev_pulp_distribution_rpm_common:
 
 dev_pulp_distribution_rpm: >-
   {%- set dev_dists = [] -%}
-  {%- for repo in rpm_package_repos -%}
+  {%- for repo in rpm_package_repos_filtered -%}
   {%- if repo.publish | default(true) -%}
   {%- set version = dev_pulp_distribution_version -%}
   {%- set name = repo.distribution_name ~ version -%}
@@ -155,7 +155,7 @@ dev_pulp_distribution_rpm_promote_common:
 
 dev_pulp_distribution_rpm_promote: >-
   {%- set dev_dists = [] -%}
-  {%- for repo in rpm_package_repos -%}
+  {%- for repo in rpm_package_repos_filtered -%}
   {%- if repo.publish | default(true) -%}
   {%- set version = dev_pulp_distribution_rpm_promote_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version -%}

ansible/inventory/group_vars/all/package-repos

@@ -79,6 +79,12 @@ deb_package_repos:
     short_name: docker_ce_ubuntu
     distribution_name: docker-ce-for-ubuntu-
 
+# Default filter string for Deb package repositories.
+deb_package_repo_filter: ""
+
+# List of package repositories after applying filter.
+deb_package_repos_filtered: "{{ deb_package_repos | select_repos(deb_package_repo_filter) }}"
+
 ###############################################################################
 # RPM
 
@@ -345,3 +351,9 @@ rpm_package_repos:
     base_path: mlnx_ofed/5.6-1.0.3.3/rhel8.6/x86_64/
     short_name: mlnx_ofed_5_6_1_0_3_3_rhel8_6
     distribution_name: mlnx_ofed-5.6-1.0.3.3-rhel8.6-
+
+# Default filter string for RPM package repositories.
+rpm_package_repo_filter: ""
+
+# List of package repositories after applying filter.
+rpm_package_repos_filtered: "{{ rpm_package_repos | select_repos(rpm_package_repo_filter) }}"

ansible/inventory/group_vars/all/test-pulp-repos

@@ -28,7 +28,7 @@ test_pulp_repository_deb_repo_versions: {}
 
 test_pulp_repository_deb_repos: >-
   {%- set test_repos = [] -%}
-  {%- for repo in deb_package_repos -%}
+  {%- for repo in deb_package_repos_filtered -%}
   {%- if repo.sync | default(true) and repo.publish | default(true) -%}
   {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%}
   {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%}
@@ -55,7 +55,7 @@ test_pulp_distribution_deb_common:
 
 test_pulp_distribution_deb: >-
   {%- set test_dists = [] -%}
-  {%- for repo in deb_package_repos -%}
+  {%- for repo in deb_package_repos_filtered -%}
   {%- if repo.sync | default(true) and repo.publish | default(true) -%}
   {%- set version = test_pulp_repository_deb_repo_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version ~ "-ark" -%}
@@ -92,7 +92,7 @@ test_pulp_repository_rpm_repo_versions: {}
 
 test_pulp_repository_rpm_repos: >-
   {%- set test_repos = [] -%}
-  {%- for repo in rpm_package_repos -%}
+  {%- for repo in rpm_package_repos_filtered -%}
   {%- if repo.sync | default(true) and repo.publish | default(true) -%}
   {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%}
   {%- set test_repo = {"name": repo.name ~ " (ark)", "url": test_dev_pulp_content_url ~ repo.base_path ~ version, "short_name": repo.short_name} -%}
@@ -110,7 +110,7 @@ test_pulp_distribution_rpm_common:
 
 test_pulp_distribution_rpm: >-
   {%- set test_dists = [] -%}
-  {%- for repo in rpm_package_repos -%}
+  {%- for repo in rpm_package_repos_filtered -%}
   {%- if repo.sync | default(true) and repo.publish | default(true) -%}
   {%- set version = test_pulp_repository_rpm_repo_versions.get(repo.short_name, omit) -%}
   {%- set name = repo.distribution_name ~ version ~ "-ark" -%}

ansible/test-kayobe-repo-version-generate.yml

@@ -17,15 +17,6 @@
     kayobe_pulp_repo_versions_path: "{{ (kayobe_config_repo_path ~ '/etc/kayobe/pulp-repo-versions.yml') | realpath }}"
     kayobe_pulp_repo_versions_dest: "{{ kayobe_pulp_repo_versions_path }}"
   tasks:
-    - name: Assert that versions variable is populated
-      assert:
-        that:
-          - missing_repos | length == 0
-        msg: >-
-          Some expected repositories not present in Kayobe Pulp repo versions: {{ missing_repos | join(',') }}
-      vars:
-        missing_repos: "{{ kayobe_pulp_repo_versions.keys() | list | difference(test_pulp_repository_rpm_repo_versions) | difference(test_pulp_repository_deb_repo_versions) | list }}"
-
     - name: Set a fact about updated Kayobe Pulp repository versions
       vars:
         kayobe_var_name: "{{ 'stackhpc_pulp_repo_' ~ item.key ~ '_version' }}"

ansible/validate-deb-repos.yml

@@ -53,6 +53,23 @@
         that:
           - published_deb_package_repos | map(attribute='base_path') | list | map('last') | unique == ['/']
 
+    - name: Assert that Deb package repository list can be filtered to single
+      assert:
+        that:
+          - deb_package_repos_filtered | length == 1
+          - deb_package_repos_filtered[0].short_name == 'ubuntu_focal'
+      vars:
+        deb_package_repo_filter: ubuntu_focal$
+
+    - name: Assert that Deb package repository list can be filtered to multiple
+      assert:
+        that:
+          - deb_package_repos_filtered | length == 2
+          - deb_package_repos_filtered[0].short_name == 'ubuntu_focal'
+          - deb_package_repos_filtered[1].short_name == 'docker_ce_ubuntu'
+      vars:
+        deb_package_repo_filter: docker_ce_ubuntu ubuntu_focal$
+
     - name: Assert that dev package repository list is defined
       assert:
         that:

ansible/validate-rpm-repos.yml

@@ -53,6 +53,23 @@
         that:
           - published_rpm_package_repos | map(attribute='base_path') | list | map('last') | unique == ['/']
 
+    - name: Assert that RPM package repository list can be filtered to single
+      assert:
+        that:
+          - rpm_package_repos_filtered | length == 1
+          - rpm_package_repos_filtered[0].short_name == 'centos_stream_8_baseos'
+      vars:
+        rpm_package_repo_filter: centos_stream_8_baseos
+
+    - name: Assert that RPM package repository list can be filtered to multiple
+      assert:
+        that:
+          - rpm_package_repos_filtered | length == 2
+          - rpm_package_repos_filtered[0].short_name == 'centos_stream_8_baseos'
+          - rpm_package_repos_filtered[1].short_name == 'docker'
+      vars:
+        rpm_package_repo_filter: docker centos_stream_8_baseos
+
     - name: Assert that dev package repository list is defined
       assert:
         that: