Matching ubi-config by major version or content set regex [RHELDST-63…

…96] (#166)

Added support for matching correct ubi-config item by
major version and/or by content set regex.
Features are enabled also for CLI.

tests/test_cli.py

@@ -71,6 +71,8 @@ def test_default_config_source(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -98,6 +100,8 @@ def test_custom_config_source(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -123,6 +127,8 @@ def test_crt(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -152,6 +158,8 @@ def test_specified_filenames(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -181,6 +189,8 @@ def test_specified_content_sets(mock_ubipopulate):
             "test_repo1-rpms",
         ],
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -210,6 +220,8 @@ def test_specified_repo_ids(mock_ubipopulate):
         repo_ids=[
             "test_repo1",
         ],
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -240,6 +252,8 @@ def test_dry_run(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -271,6 +285,8 @@ def test_custom_workers_number(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -303,6 +319,8 @@ def test_insecure(mock_ubipopulate):
         None,
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
     )
 
 
@@ -330,4 +348,64 @@ def test_output_file(mock_ubipopulate):
         "/foo/out/repos.txt",
         content_sets=None,
         repo_ids=None,
+        version=None,
+        content_set_regex=None,
+    )
+
+
+@mock.patch("ubipop.UbiPopulate")
+def test_version(mock_ubipopulate):
+    args = [
+        "--pulp-hostname",
+        "foo.pulp.com",
+        "--user",
+        "foo",
+        "--pass",
+        "foo",
+        "--version",
+        "100",
+    ]
+    main(args)
+    mock_ubipopulate.assert_called_once_with(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        [],
+        None,
+        False,
+        4,
+        None,
+        content_sets=None,
+        repo_ids=None,
+        version="100",
+        content_set_regex=None,
+    )
+
+
+@mock.patch("ubipop.UbiPopulate")
+def test_content_set_regex(mock_ubipopulate):
+    args = [
+        "--pulp-hostname",
+        "foo.pulp.com",
+        "--user",
+        "foo",
+        "--pass",
+        "foo",
+        "--content-set-regex",
+        "test-regex.*",
+    ]
+    main(args)
+    mock_ubipopulate.assert_called_once_with(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        [],
+        None,
+        False,
+        4,
+        None,
+        content_sets=None,
+        repo_ids=None,
+        version=None,
+        content_set_regex="test-regex.*",
     )

tests/test_ubipop.py

@@ -518,6 +518,112 @@ def test_load_ubiconfig_by_repo_ids():
     )
 
 
+def test_load_ubiconfig_by_version():
+    """Ensure correct config is returned when given a major version of ubi"""
+    repo = YumRepository(id="rhel-7-server", content_set="rhel-7-server-rpms")
+
+    fake_ubipopulate = FakeUbiPopulate(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        ubiconfig_dir_or_url=TEST_DATA_DIR,
+        version="7",
+    )
+
+    fake_pulp = fake_ubipopulate.pulp_client_controller
+    fake_pulp.insert_repository(repo)
+
+    assert len(fake_ubipopulate.ubiconfig_list) == 1
+    assert (
+        fake_ubipopulate.ubiconfig_list[0].content_sets.rpm.output
+        == "ubi-7-server-rpms"
+    )
+
+
+def test_load_ubiconfig_by_version_no_match():
+    """Ensure no config is returned when given a major version of ubi that doesn't exist in data"""
+    repo = YumRepository(id="rhel-7-server", content_set="rhel-7-server-rpms")
+
+    fake_ubipopulate = FakeUbiPopulate(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        ubiconfig_dir_or_url=TEST_DATA_DIR,
+        version="10",
+    )
+
+    fake_pulp = fake_ubipopulate.pulp_client_controller
+    fake_pulp.insert_repository(repo)
+
+    # no config should match
+    assert len(fake_ubipopulate.ubiconfig_list) == 0
+
+
+def test_load_ubiconfig_by_content_set_regex():
+    """Ensure correct config is returned when given a content set regex"""
+    repo = YumRepository(
+        id="ubi-8-for-x86_64-appstream-rpms__8",
+        content_set="ubi-8-for-x86_64-appstream-rpms",
+    )
+
+    fake_ubipopulate = FakeUbiPopulate(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        ubiconfig_dir_or_url=TEST_DATA_DIR,
+        content_set_regex="ubi-8.*",
+    )
+
+    fake_pulp = fake_ubipopulate.pulp_client_controller
+    fake_pulp.insert_repository(repo)
+
+    # two config files are matched according to testdata
+    assert len(fake_ubipopulate.ubiconfig_list) == 2
+
+    fake_ubipopulate.ubiconfig_list.sort(key=lambda x: x.file_name)
+    conf_1 = fake_ubipopulate.ubiconfig_list[0]
+    conf_2 = fake_ubipopulate.ubiconfig_list[1]
+
+    assert conf_1.file_name == "ubiconf_golang.yaml"
+    assert conf_1.content_sets.rpm.output == "ubi-8-for-x86_64-appstream-rpms"
+
+    assert conf_2.file_name == "ubiconf_golang2.yaml"
+    assert conf_2.content_sets.rpm.output == "ubi-8-for-x86_64-appstream-rpms"
+
+
+def test_load_ubiconfig_by_content_set_regex_and_version():
+    """Ensure correct config is returned when given a content set regex and version"""
+    repo = YumRepository(
+        id="ubi-8-for-x86_64-appstream-rpms__8",
+        content_set="ubi-8-for-x86_64-appstream-rpms",
+    )
+
+    fake_ubipopulate = FakeUbiPopulate(
+        "foo.pulp.com",
+        ("foo", "foo"),
+        False,
+        ubiconfig_dir_or_url=TEST_DATA_DIR,
+        version="8",
+        content_set_regex="x86_64",
+    )
+
+    fake_pulp = fake_ubipopulate.pulp_client_controller
+    fake_pulp.insert_repository(repo)
+
+    # two config files are matched according to testdata
+    assert len(fake_ubipopulate.ubiconfig_list) == 2
+
+    fake_ubipopulate.ubiconfig_list.sort(key=lambda x: x.file_name)
+    conf_1 = fake_ubipopulate.ubiconfig_list[0]
+    conf_2 = fake_ubipopulate.ubiconfig_list[1]
+
+    assert conf_1.file_name == "ubiconf_golang.yaml"
+    assert conf_1.content_sets.rpm.output == "ubi-8-for-x86_64-appstream-rpms"
+
+    assert conf_2.file_name == "ubiconf_golang2.yaml"
+    assert conf_2.content_sets.rpm.output == "ubi-8-for-x86_64-appstream-rpms"
+
+
 @pytest.fixture(name="mocked_ubiconfig_load_all")
 def fixture_mocked_ubiconfig_load_all():
     with patch("ubiconfig.get_loader") as get_loader:

ubipop/__init__.py

@@ -117,25 +117,10 @@ def __init__(
         self._ubiconfig_dir_or_url = ubiconfig_dir_or_url
         self._content_sets = kwargs.get("content_sets", None)
         self._repo_ids = kwargs.get("repo_ids", None)
+        self._version = kwargs.get("version", None)
+        self._content_set_regex = kwargs.get("content_set_regex", None)
         self._ubiconfig_map = None
 
-    @property
-    def ubiconfig_list(self):
-        if self._ubiconfig_list is None:
-            self._ubiconfig_list = self._load_ubiconfig(
-                self._ubiconfig_filename_list,
-                self._ubiconfig_dir_or_url,
-                self._content_sets,
-                self._repo_ids,
-            )
-        return self._ubiconfig_list
-
-    @property
-    def ubiconfig_map(self):
-        if self._ubiconfig_map is None:
-            self._ubiconfig_map = self._create_config_map()
-        return self._ubiconfig_map
-
     @property
     def pulp_client(self):
         if self._pulp_client is None:
@@ -147,9 +132,15 @@ def pulp_client(self):
     def _make_pulp_client(self, url, auth, insecure):
         return Client("https://" + url, auth=auth, verify=not insecure)
 
-    def _load_ubiconfig(
-        self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None
-    ):
+    @property
+    def ubiconfig_list(self):
+        if self._ubiconfig_list is None:
+            self._ubiconfig_list = self._load_ubiconfig()
+        return self._ubiconfig_list
+
+    def _load_ubiconfig(self):
+        ubiconfig_dir_or_url = self._ubiconfig_dir_or_url
+        filenames = self._ubiconfig_filename_list
         loader = ubiconfig.get_loader(ubiconfig_dir_or_url)
         ubi_conf_list = []
 
@@ -158,29 +149,49 @@ def _load_ubiconfig(
         else:
             ubi_conf_list.extend(loader.load_all())
 
-        return self._filter_ubi_conf_list(ubi_conf_list, content_sets, repo_ids)
+        return self._filter_ubi_conf_list(ubi_conf_list)
 
-    def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
+    @property
+    def ubiconfig_map(self):
+        if self._ubiconfig_map is None:
+            self._ubiconfig_map = self._create_config_map()
+        return self._ubiconfig_map
+
+    def _filter_ubi_conf_list(self, config_list):
         """
         Reduces the list of UBI configurations to only those matching
-        provided content sets and/or repo IDs.
+        provided content sets and/or repo IDs. Config can be further filter
+        by major version of or content_set_regex.
         """
 
+        content_sets = self._content_sets
+        repo_ids = self._repo_ids
+        version = self._version
+        content_set_regex = self._content_set_regex
+
         filtered_conf_list = []
 
         content_sets = content_sets or []
         repo_ids = repo_ids or []
 
-        if not content_sets and not repo_ids:
+        if not any([content_sets, repo_ids, version, content_set_regex]):
+            # no filtering requested, return complete list of ubi config
             return config_list
 
+        if content_set_regex:
+            content_set_regex = re.compile(content_set_regex)
+
         if repo_ids:
             repos = [self.pulp_client.get_repository(repo_id) for repo_id in repo_ids]
 
             for repo in repos:
                 content_sets.append(repo.content_set)
 
         for conf in config_list:
+            # if requested, filter config by major version
+            if version and not conf.version.startswith(version):
+                continue
+
             for label in [
                 conf.content_sets.rpm.input,
                 conf.content_sets.rpm.output,
@@ -189,7 +200,16 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
                 conf.content_sets.debuginfo.input,
                 conf.content_sets.debuginfo.output,
             ]:
-                if label in content_sets:
+
+                # matching by specific content sets takes precedence
+                # or if empty content_set, content_set_regex - take the config immediately
+                # we don't have any other filter to use
+                if (
+                    not any([content_sets, content_set_regex])
+                    or label in content_sets
+                    or (content_set_regex and re.search(content_set_regex, label))
+                ):
+
                     filtered_conf_list.append(conf)
                     break