Merge ce3922a into d12ae6a

tests/test_pulp.py

@@ -43,6 +43,7 @@ def fixture_search_repo_response():
             "content_set": "test_repo-source-rpms",
             "platform": "ubi",
             "platform_full_version": "7",
+            "ubi_config_version": "7",
         },
         "distributors": [{
             "id": "dist_id",
@@ -58,6 +59,7 @@ def fixture_mock_repo():
         arch="x86_64",
         content_set="test_repo-source-rpms",
         platform_full_version="7",
+        ubi_config_version="7",
         dist_ids_type_ids=[
             ("dist_id_1", "dist_type_id_1"),
             ("dist_id_2", "dist_type_id_2"),

tests/test_ubipop.py

@@ -47,8 +47,9 @@ def fixture_ubi_repo_set_no_debug():
 
 @pytest.fixture(name='test_ubiconfig')
 def fixture_test_ubiconfig():
-    yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
-
+    config = ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml")
+    config.version = '7.7'
+    yield config
 
 @pytest.fixture(name='executor')
 def fixture_executor():
@@ -68,6 +69,7 @@ def get_test_repo(**kwargs):
         kwargs.get('platform_full_version'),
         kwargs.get('distributors_ids_type_ids'),
         kwargs.get('ubi_population'),
+        kwargs.get('ubi_config_version'),
     )
 
 
@@ -725,15 +727,18 @@ def test_ubi_repo_set(rhel_repo_set, ubi_repo_set, fail, caplog):
 @pytest.fixture(name='mocked_ubiconfig_load')
 def fixture_mocked_ubiconfig_load():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load.return_value = "test"
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7.7"
+        get_loader.return_value.load.return_value = m
         yield get_loader
 
 
 def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ['cfg.yaml'])
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_load_ubiconfig_by_content_set_labels():
@@ -759,15 +764,18 @@ def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id):
 @pytest.fixture(name='mocked_ubiconfig_load_all')
 def fixture_mocked_ubiconfig_load_all():
     with patch('ubiconfig.get_loader') as get_loader:
-        get_loader.return_value.load_all.return_value = ["test"]
+        m = MagicMock()
+        m.file_name = "test"
+        m.version = "7.7"
+        get_loader.return_value.load_all.return_value = [m]
         yield get_loader
 
 
 def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all):
     # pylint: disable=unused-argument
     ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False)
     assert len(ubipop.ubiconfig_list) == 1
-    assert ubipop.ubiconfig_list[0] == "test"
+    assert ubipop.ubiconfig_list[0].file_name == "test"
 
 
 def test_create_srpms_output_set(mock_ubipop_runner):

tests/test_utils.py

@@ -16,15 +16,15 @@
 
 def test_raise_not_implemented_pulp_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "test-rpms", "2", None, None)
+    repo = Repo("test", "1", "test-rpms", "2", "2", None, None)
     action = PulpAction(units, repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
 
 def test_raise_not_implemented_associate_action():
     units = ["unit1", "unit2"]
-    repo = Repo("test", "1", "test-rpms", "2", None, None)
-    src_repo = Repo("test", "1", "test-rpms", "2", None, None)
+    repo = Repo("test", "1", "test-rpms", "2", "2", None, None)
+    src_repo = Repo("test", "1", "test-rpms", "2", "2", None, None)
     action = AssociateAction(units, repo, src_repo)
     pytest.raises(NotImplementedError, action.get_action, None)
 
@@ -36,8 +36,8 @@ def test_raise_not_implemented_associate_action():
 ])
 def test_get_action_associate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None)
-    src_repo = Repo("test_src", "1", "test_src-rpms", "2", None, None)
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None)
+    src_repo = Repo("test_src", "1", "test_src-rpms", "2", "2", None, None)
     action = klass(units, dst_repo, src_repo)
     associate_action, src_repo_current, dst_repo_current, current_units = \
         action.get_action(MagicMock())
@@ -55,7 +55,7 @@ def test_get_action_associate(klass, method):
 ])
 def test_get_action_unassociate(klass, method):
     units = ["unit1", "unit2"]
-    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None)
+    dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None)
     action = klass(units, dst_repo)
     associate_action, dst_repo_current, current_units = action.get_action(MagicMock())
 

ubipop/__init__.py

@@ -26,6 +26,10 @@ class RepoMissing(Exception):
     pass
 
 
+class ConfigMissing(Exception):
+    pass
+
+
 RepoSet = namedtuple('RepoSet', ['rpm', 'source', 'debug'])
 
 
@@ -90,6 +94,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No
         self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
                                                    content_sets=kwargs.get('content_sets', None),
                                                    repo_ids=kwargs.get('repo_ids', None))
+        self.ubiconfig_map = self._create_config_map()
 
     def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None):
         loader = ubiconfig.get_loader(ubiconfig_dir_or_url)
@@ -132,18 +137,83 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids):
 
         return filtered_conf_list
 
+    def _create_config_map(self):
+        """Create a config map from self.ubiconfig_list, it has the form in:
+            {
+                "7.7":
+                    {
+                        "config_filename1": config1,
+                        "config_filename2": config2,
+                        ...,
+                    },
+                "8.1":
+                    {
+                        "config_filename1": config1,
+                        ...,
+                    },
+                ....
+            }
+        """
+
+        config_map = {}
+        for config in self.ubiconfig_list:
+            config_map.setdefault(config.version, {})\
+                .setdefault(config.file_name, config)
+
+        return config_map
+
     def populate_ubi_repos(self):
         out_repos = set()
-
+        used_content_sets = set()
+        # since repos are searched by content sets, same repo could be searched and populated
+        # multiple times, to avoid that, cache the content sets already used and skip the config
+        # whose content sets are all in the cache
         for config in self.ubiconfig_list:
+            content_sets = [
+                config.content_sets.rpm,
+                config.content_sets.srpm,
+                config.content_sets.debuginfo
+            ]
+            to_use = [cs for cs in content_sets if cs not in used_content_sets]
+            if to_use:
+                for cs in to_use:
+                    used_content_sets.add(cs)
+            else:
+                _LOG.debug("Skipping %s, since it's been used already", config.file_name)
+                continue
+
             try:
                 repo_pairs = self._get_input_and_output_repo_pairs(config)
             except RepoMissing:
                 _LOG.warning("Skipping current content triplet, some repos are missing")
                 continue
 
             for repo_set in repo_pairs:
-                UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run,
+                ubi_config_version = repo_set.out_repos.rpm.ubi_config_version
+                platform_full_version = repo_set.out_repos.rpm.platform_full_version
+                # get the right config file by ubi_config_version attr, if it's None,
+                # then it's not a mainline repo, use platform_full_version instead.
+                # config file could also be missing for specific version, then the
+                # default config file will be used.
+                version = ubi_config_version or platform_full_version
+                right_config = self.ubiconfig_map\
+                    .get(version, {})\
+                    .get(config.file_name)\
+                    or self.ubiconfig_map\
+                        .get(platform_full_version, {})\
+                        .get(config.file_name)
+
+                # if config file is missing from wanted version, as well as default
+                # branch, raise exception
+                if not right_config:
+                    _LOG.error(
+                        'Config file %s missing from %s and default %s branches',
+                        config.file_name,
+                        version,
+                        platform_full_version,
+                    )
+
+                UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run,
                                   self._executor).run_ubi_population()
 
                 out_repos.update(repo_set.get_output_repo_ids())

ubipop/_pulp_client.py

@@ -90,13 +90,15 @@ def _search_repo(self, criteria):
             # Only UBI repos should have a ubi_population note, set to None for other platforms
             # If the UBI repo does not have this note, assume it is okay to populate
             ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None
+            ubi_config_version = notes.get('ubi_config_version', None)
             repos.append(Repo(
                 repo_id=item['id'],
                 arch=notes['arch'],
                 content_set=notes['content_set'],
                 platform_full_version=notes['platform_full_version'],
                 dist_ids_type_ids=dist_info,
                 ubi_population=ubi_population,
+                ubi_config_version=ubi_config_version,
             ))
 
         return repos
@@ -311,11 +313,12 @@ def publish_repo(self, repo):
 
 class Repo(object):
     def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids,
-                 ubi_population):
+                 ubi_population, ubi_config_version):
         self.repo_id = repo_id
         self.arch = arch
         self.content_set = content_set
         self.platform_full_version = platform_full_version
+        self.ubi_config_version = ubi_config_version
         self.distributors_ids_type_ids_tuples = dist_ids_type_ids
         self.ubi_population = ubi_population