Merge c7a1689 into f7ef877

requirements.txt

@@ -1,4 +1,4 @@
 requests
 more-executors
-ubi-config
+ubi-config>=2.1.0
 rpm-py-installer

tests/integration/test_association.py

@@ -19,7 +19,7 @@
 PULP_SECURE = os.getenv('TEST_PULP_SECURE', '0').lower() in ['true', 'yes', '1']
 PULP_RPM_REPO_PREFIX = '/pulp/rpmrepos/'
 
-TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data')
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data/ubi7')
 
 INTEGRATION_NOT_SETUP = PULP_HOSTNAME is None
 

tests/test_pulp.py

@@ -43,6 +43,7 @@ def fixture_search_repo_response():
             "content_set": "test_repo-source-rpms",
             "platform": "ubi",
             "platform_full_version": "7",
+            "ubi_config_version": "7",
         },
         "distributors": [{
             "id": "dist_id",
@@ -58,6 +59,7 @@ def fixture_mock_repo():
         arch="x86_64",
         content_set="test_repo-source-rpms",
         platform_full_version="7",
+        ubi_config_version="7",
         dist_ids_type_ids=[
             ("dist_id_1", "dist_type_id_1"),
             ("dist_id_2", "dist_type_id_2"),

tests/test_ubipop.py

@@ -27,12 +27,18 @@
 
 @pytest.fixture(name='ubi_repo_set')
 def fixture_ubi_repo_set():
-    yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms")],
-                             [get_test_repo(repo_id="foo-source")],
-                             [get_test_repo(repo_id="foo-debug")]),
-                     RepoSet(get_test_repo(repo_id="ubi-foo-rpms"),
-                             get_test_repo(repo_id="ubi-foo-source"),
-                             get_test_repo(repo_id="ubi-foo-debug")))
+    yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms",
+                                            ubi_config_version="7")],
+                             [get_test_repo(repo_id="foo-source",
+                                            ubi_config_version="7")],
+                             [get_test_repo(repo_id="foo-debug",
+                                            ubi_config_version="7")]),
+                     RepoSet(get_test_repo(repo_id="ubi-foo-rpms",
+                                           ubi_config_version="7"),
+                             get_test_repo(repo_id="ubi-foo-source",
+                                           ubi_config_version="7"),
+                             get_test_repo(repo_id="ubi-foo-debug",
+                                           ubi_config_version="7")))
 
 
 @pytest.fixture(name='ubi_repo_set_no_debug')
@@ -47,7 +53,7 @@ def fixture_ubi_repo_set_no_debug():
 
 @pytest.fixture(name='test_ubiconfig')
 def fixture_test_ubiconfig():
-    yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
+    yield ubiconfig.get_loader(TEST_DATA_DIR).load("ubi7/conf.yaml")
 
 
 @pytest.fixture(name='executor')
@@ -69,6 +75,7 @@ def get_test_repo(**kwargs):
         kwargs.get('distributors_ids_type_ids'),
         kwargs.get('ubi_population'),
         kwargs.get('population_sources'),
+        kwargs.get('ubi_config_version'),
     )
 
 
@@ -118,21 +125,49 @@ def test_get_output_repo_ids_no_debug(ubi_repo_set_no_debug):
 def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog):
     # Don't actually query Pulp for repos
     mocked_search_repo_by_cs.side_effect = [
+        # Output repos - rhel-7-server
+        [get_test_repo(
+            repo_id="ubi-7-server-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-rpms",
+            ubi_population=False,
+            platform_full_version="7.2",
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-source-rpms",
+            ubi_population=True,
+            # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo
+            # for skipping whole repo triplet
+            platform_full_version="7.2",
+        ), ],
+        [get_test_repo(
+            repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64",
+            content_set="ubi-7-server-debuginfo-rpms",
+            ubi_population=False,
+            platform_full_version="7.2"
+        ), ],
+
         # Output repos - rhel-8-for-x86_64-appstream
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-rpms",
             content_set="ubi-8-for-x86_64-appstream-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-source-rpms",
             content_set="ubi-8-for-x86_64-appstream-source-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
         [get_test_repo(
             repo_id="ubi-8-for-x86_64-appstream-debug-rpms",
             content_set="ubi-8-for-x86_64-appstream-debug-rpms",
-            ubi_population=True
+            ubi_population=True,
+            ubi_config_version="8",
+            platform_full_version="8"
         ), ],
 
         # Input repos - rhel-8-for-x86_64-appstream
@@ -148,39 +183,6 @@ def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner,
             repo_id="rhel-8-for-x86_64-appstream-debug-rpms",
             content_set="rhel-8-for-x86_64-appstream-debug-rpms",
         ), ],
-
-        # Output repos - rhel-7-server
-        [get_test_repo(
-            repo_id="ubi-7-server-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-rpms",
-            ubi_population=False
-        ), ],
-        [get_test_repo(
-            repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-source-rpms",
-            ubi_population=True
-            # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo
-            # for skipping whole repo triplet
-        ), ],
-        [get_test_repo(
-            repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64",
-            content_set="ubi-7-server-debuginfo-rpms",
-            ubi_population=False
-        ), ],
-
-        # Input repos - rhel-7-server
-        [get_test_repo(
-            repo_id="rhel-7-server-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-rpms",
-        ), ],
-        [get_test_repo(
-            repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-source-rpms",
-        ), ],
-        [get_test_repo(
-            repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64",
-            content_set="rhel-7-server-debuginfo-rpms",
-        ), ],
     ]
 
     # Attempt to populate both invalid and valid repo sets
@@ -522,7 +524,7 @@ def test_match_modules(mock_ubipop_runner):
 
 
 def test_match_modules_without_profile(ubi_repo_set, executor):
-    test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubiconf_golang.yaml')
+    test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubi8/ubiconf_golang.yaml')
     mocked_ubipop_runner = UbiPopulateRunner(
         MagicMock(),
         ubi_repo_set,
@@ -771,15 +773,18 @@ def test_ubi_repo_set(rhel_repo_set, ubi_repo_set, fail, caplog):
 @pytest.fixture(name='mocked_ubiconfig_load')
 def fixture_mocked_ubiconfig_load():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load.return_value = "test"
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7.7"
+        get_loader.return_value.load.return_value = m
         yield get_loader
 
 
 def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ['cfg.yaml'])
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_load_ubiconfig_by_content_set_labels():
@@ -805,15 +810,18 @@ def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id):
 @pytest.fixture(name='mocked_ubiconfig_load_all')
 def fixture_mocked_ubiconfig_load_all():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load_all.return_value = ["test"]
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7"
+        get_loader.return_value.load_all.return_value = [m]
         yield get_loader
 
 
 def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False)
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_create_srpms_output_set(mock_ubipop_runner):

tests/test_utils.py

@@ -16,15 +16,15 @@
 
 def test_raise_not_implemented_pulp_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "test-rpms", "2", None, None, None)
+    repo = Repo("test", "1", "test-rpms", "2", "2", None, None, None)
     action = PulpAction(units, repo)
     pytest.raises(NotImplementedError, action.get_actions, None)
 
 
 def test_raise_not_implemented_associate_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "test-rpms", "2", None, None, None)
-    src_repo = Repo("test", "1", "test-rpms", "2", None, None, None)
+    repo = Repo("test", "1", "test-rpms", "2", "2", None, None, None)
+    src_repo = Repo("test", "1", "test-rpms", "2", "2", None, None, None)
     action = AssociateAction(units, repo, src_repo)
     pytest.raises(NotImplementedError, action.get_actions, None)
 
@@ -40,12 +40,11 @@ def test_get_action_associate(klass, method):
     mocked_unit_2 = MagicMock()
     mocked_unit_2.associate_source_repo_id = "test_src_2"
     units = [mocked_unit_1, mocked_unit_2]
-    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None)
-    src_repos = [Repo("test_src_1", "1", "test_src-rpms", "2", None, None, None),
-                 Repo("test_src_2", "1", "test_src-rpms", "2", None, None, None)]
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None, None)
+    src_repos = [Repo("test_src_1", "1", "test_src-rpms", "2", "2", None, None, None),
+                 Repo("test_src_2", "1", "test_src-rpms", "2", "2", None, None, None)]
     action = klass(units, dst_repo, src_repos)
     actions = action.get_actions(MagicMock())
-
     for action in actions:
         associate_action, src_repo_current, dst_repo_current, current_units = action
         assert "mock." + method in str(associate_action)
@@ -63,7 +62,7 @@ def test_get_action_associate(klass, method):
 ])
 def test_get_action_unassociate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None)
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None, None)
     action = klass(units, dst_repo)
     associate_action, dst_repo_current, current_units = action.get_actions(MagicMock())[0]
 

ubipop/__init__.py

@@ -26,6 +26,10 @@ class RepoMissing(Exception):
     pass
 
 
+class ConfigMissing(Exception):
+    pass
+
+
 RepoSet = namedtuple('RepoSet', ['rpm', 'source', 'debug'])
 
 
@@ -90,6 +94,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No
         self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
                                                    content_sets=kwargs.get('content_sets', None),
                                                    repo_ids=kwargs.get('repo_ids', None))
+        self.ubiconfig_map = self._create_config_map()
 
     def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None):
         loader = ubiconfig.get_loader(ubiconfig_dir_or_url)
@@ -132,10 +137,52 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
 
         return filtered_conf_list
 
+    def _create_config_map(self):
+        """Create a config map from self.ubiconfig_list, it has the form in:
+            {
+                "7.7":
+                    {
+                        "config_filename1": config1,
+                        "config_filename2": config2,
+                        ...,
+                    },
+                "8.1":
+                    {
+                        "config_filename1": config1,
+                        ...,
+                    },
+                ....
+            }
+        """
+
+        config_map = {}
+        for config in self.ubiconfig_list:
+            config_map.setdefault(config.version, {})\
+                .setdefault(config.file_name, config)
+
+        return config_map
+
     def populate_ubi_repos(self):
         out_repos = set()
+        used_content_sets = set()
+        # since repos are searched by content sets, same repo could be searched and populated
+        # multiple times, to avoid that, cache the content sets already used and skip the config
+        # whose content sets are all in the cache
+
+        for config in sorted(self.ubiconfig_list, key=str):
+            content_sets = [
+                config.content_sets.rpm,
+                config.content_sets.srpm,
+                config.content_sets.debuginfo
+            ]
+            to_use = [cs for cs in content_sets if cs not in used_content_sets]
+            if to_use:
+                for cs in to_use:
+                    used_content_sets.add(cs)
+            else:
+                _LOG.debug("Skipping %s, since it's been used already", config.file_name)
+                continue
 
-        for config in self.ubiconfig_list:
             try:
                 repo_pairs = self._get_ubi_repo_sets(config)
 
@@ -144,7 +191,32 @@ def populate_ubi_repos(self):
                 continue
 
             for repo_set in repo_pairs:
-                UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run,
+                ubi_config_version = repo_set.out_repos.rpm.ubi_config_version
+                platform_full_version = repo_set.out_repos.rpm.platform_full_version
+                # get the right config file by ubi_config_version attr, if it's None,
+                # then it's not a mainline repo, use platform_full_version instead.
+                # config file could also be missing for specific version, then the
+                # default config file will be used.
+                version = ubi_config_version or platform_full_version
+                right_config = self.ubiconfig_map\
+                    .get(str(version), {})\
+                    .get(config.file_name)\
+                    or self.ubiconfig_map\
+                        .get(str(platform_full_version), {})\
+                        .get(config.file_name)
+
+                # if config file is missing from wanted version, as well as default
+                # branch, raise exception
+                if not right_config:
+                    _LOG.error(
+                        'Config file %s missing from %s and default %s branches',
+                        config.file_name,
+                        version,
+                        platform_full_version,
+                    )
+                    raise ConfigMissing()
+
+                UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run,
                                   self._executor).run_ubi_population()
 
                 out_repos.update(repo_set.get_output_repo_ids())
@@ -158,7 +230,7 @@ def _get_ubi_repo_sets(self, ubi_config_item):
         """
         Searches for ubi repository triplet (binary rpm, srpm, debug) for
         one ubi config item and tries to determine their population sources
-        (input repositories). Returns list UbiRepoSet objects that provides 
+        (input repositories). Returns list UbiRepoSet objects that provides
         input and output repositories that are used for population process.
         """
         rpm_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs,