diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 7f7862f..4123b11 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -43,6 +43,7 @@ def fixture_search_repo_response(): "content_set": "test_repo-source-rpms", "platform": "ubi", "platform_full_version": "7", + "ubi_config_version": "7", }, "distributors": [{ "id": "dist_id", @@ -58,6 +59,7 @@ def fixture_mock_repo(): arch="x86_64", content_set="test_repo-source-rpms", platform_full_version="7", + ubi_config_version="7", dist_ids_type_ids=[ ("dist_id_1", "dist_type_id_1"), ("dist_id_2", "dist_type_id_2"), diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 346fd94..52d49e8 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -47,8 +47,9 @@ def fixture_ubi_repo_set_no_debug(): @pytest.fixture(name='test_ubiconfig') def fixture_test_ubiconfig(): - yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml") - + config = ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml") + config.version = '7.7' + yield config @pytest.fixture(name='executor') def fixture_executor(): @@ -68,6 +69,7 @@ def get_test_repo(**kwargs): kwargs.get('platform_full_version'), kwargs.get('distributors_ids_type_ids'), kwargs.get('ubi_population'), + kwargs.get('ubi_config_version'), ) @@ -725,7 +727,10 @@ def test_ubi_repo_set(rhel_repo_set, ubi_repo_set, fail, caplog): @pytest.fixture(name='mocked_ubiconfig_load') def fixture_mocked_ubiconfig_load(): with patch('ubiconfig.get_loader') as get_loader: - get_loader.return_value.load.return_value = "test" + m = MagicMock() + m.file_name = "test" + m.version = "7.7" + get_loader.return_value.load.return_value = m yield get_loader @@ -733,7 +738,7 @@ def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load): # pylint: disable=unused-argument ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ['cfg.yaml']) assert len(ubipop.ubiconfig_list) == 1 - assert ubipop.ubiconfig_list[0] == "test" + assert ubipop.ubiconfig_list[0].file_name == "test" def test_load_ubiconfig_by_content_set_labels(): @@ -759,7 +764,10 @@ def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id): @pytest.fixture(name='mocked_ubiconfig_load_all') def fixture_mocked_ubiconfig_load_all(): with patch('ubiconfig.get_loader') as get_loader: - get_loader.return_value.load_all.return_value = ["test"] + m = MagicMock() + m.file_name = "test" + m.version = "7.7" + get_loader.return_value.load_all.return_value = [m] yield get_loader @@ -767,7 +775,7 @@ def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all): # pylint: disable=unused-argument ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False) assert len(ubipop.ubiconfig_list) == 1 - assert ubipop.ubiconfig_list[0] == "test" + assert ubipop.ubiconfig_list[0].file_name == "test" def test_create_srpms_output_set(mock_ubipop_runner): diff --git a/tests/test_utils.py b/tests/test_utils.py index ee39e1b..61e9a88 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,15 +16,15 @@ def test_raise_not_implemented_pulp_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None) + repo = Repo("test", "1", "test-rpms", "2", "2", None, None) action = PulpAction(units, repo) pytest.raises(NotImplementedError, action.get_action, None) def test_raise_not_implemented_associate_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None) - src_repo = Repo("test", "1", "test-rpms", "2", None, None) + repo = Repo("test", "1", "test-rpms", "2", "2", None, None) + src_repo = Repo("test", "1", "test-rpms", "2", "2", None, None) action = AssociateAction(units, repo, src_repo) pytest.raises(NotImplementedError, action.get_action, None) @@ -36,8 +36,8 @@ def test_raise_not_implemented_associate_action(): ]) def test_get_action_associate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None) - src_repo = Repo("test_src", "1", "test_src-rpms", "2", None, None) + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None) + src_repo = Repo("test_src", "1", "test_src-rpms", "2", "2", None, None) action = klass(units, dst_repo, src_repo) associate_action, src_repo_current, dst_repo_current, current_units = \ action.get_action(MagicMock()) @@ -55,7 +55,7 @@ def test_get_action_associate(klass, method): ]) def test_get_action_unassociate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None) + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", None, None) action = klass(units, dst_repo) associate_action, dst_repo_current, current_units = action.get_action(MagicMock()) diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 4d3582e..1fbc2ea 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -26,6 +26,10 @@ class RepoMissing(Exception): pass +class ConfigMissing(Exception): + pass + + RepoSet = namedtuple('RepoSet', ['rpm', 'source', 'debug']) @@ -90,6 +94,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, content_sets=kwargs.get('content_sets', None), repo_ids=kwargs.get('repo_ids', None)) + self.ubiconfig_map = self._create_config_map() def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None): loader = ubiconfig.get_loader(ubiconfig_dir_or_url) @@ -132,10 +137,51 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids): return filtered_conf_list + def _create_config_map(self): + """Create a config map from self.ubiconfig_list, it has the form in: + { + "7.7": + { + "config_filename1": config1, + "config_filename2": config2, + ..., + }, + "8.1": + { + "config_filename1": config1, + ..., + }, + .... + } + """ + + config_map = {} + for config in self.ubiconfig_list: + config_map.setdefault(config.version, {})\ + .setdefault(config.file_name, config) + + return config_map + def populate_ubi_repos(self): out_repos = set() - + used_content_sets = set() + # since repos are searched by content sets, same repo could be searched and populated + # multiple times, to avoid that, cache the content sets already used and skip the config + # whose content sets are all in the cache for config in self.ubiconfig_list: + content_sets = [ + config.content_sets.rpm, + config.content_sets.srpm, + config.content_sets.debuginfo + ] + to_use = [cs for cs in content_sets if cs not in used_content_sets] + if to_use: + for cs in to_use: + used_content_sets.add(cs) + else: + _LOG.debug("Skipping %s, since it's been used already", config.file_name) + continue + try: repo_pairs = self._get_input_and_output_repo_pairs(config) except RepoMissing: @@ -143,7 +189,31 @@ def populate_ubi_repos(self): continue for repo_set in repo_pairs: - UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run, + ubi_config_version = repo_set.out_repos.rpm.ubi_config_version + platform_full_version = repo_set.out_repos.rpm.platform_full_version + # get the right config file by ubi_config_version attr, if it's None, + # then it's not a mainline repo, use platform_full_version instead. + # config file could also be missing for specific version, then the + # default config file will be used. + version = ubi_config_version or platform_full_version + right_config = self.ubiconfig_map\ + .get(version, {})\ + .get(config.file_name)\ + or self.ubiconfig_map\ + .get(platform_full_version, {})\ + .get(config.file_name) + + # if config file is missing from wanted version, as well as default + # branch, raise exception + if not right_config: + _LOG.error( + 'Config file %s missing from %s and default %s branches', + config.file_name, + version, + platform_full_version, + ) + + UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run, self._executor).run_ubi_population() out_repos.update(repo_set.get_output_repo_ids()) diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 98e6958..0768908 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -90,6 +90,7 @@ def _search_repo(self, criteria): # Only UBI repos should have a ubi_population note, set to None for other platforms # If the UBI repo does not have this note, assume it is okay to populate ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None + ubi_config_version = notes.get('ubi_config_version', None) repos.append(Repo( repo_id=item['id'], arch=notes['arch'], @@ -97,6 +98,7 @@ def _search_repo(self, criteria): platform_full_version=notes['platform_full_version'], dist_ids_type_ids=dist_info, ubi_population=ubi_population, + ubi_config_version=ubi_config_version, )) return repos @@ -311,11 +313,12 @@ def publish_repo(self, repo): class Repo(object): def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids, - ubi_population): + ubi_population, ubi_config_version): self.repo_id = repo_id self.arch = arch self.content_set = content_set self.platform_full_version = platform_full_version + self.ubi_config_version = ubi_config_version self.distributors_ids_type_ids_tuples = dist_ids_type_ids self.ubi_population = ubi_population