From 9b8a9666bd80a55651f7402eb70c923d2255a875 Mon Sep 17 00:00:00 2001 From: Jay Zhang Date: Tue, 15 Oct 2019 16:47:25 -0400 Subject: [PATCH] Use correct version of ubi config file for population This resloves #138 A new attr is added to Repo class and it will be populated while searching repos. After all config files are loaded, it now creates a config map, which helps find the right config file for the repo to populate. --- tests/test_pulp.py | 1 + tests/test_ubipop.py | 6 +++-- ubipop/__init__.py | 52 ++++++++++++++++++++++++++++++++++++++++-- ubipop/_pulp_client.py | 5 +++- 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 7f7862f..6a33557 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -43,6 +43,7 @@ def fixture_search_repo_response(): "content_set": "test_repo-source-rpms", "platform": "ubi", "platform_full_version": "7", + "ubi_config_version": "7", }, "distributors": [{ "id": "dist_id", diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 346fd94..df29e40 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -47,8 +47,9 @@ def fixture_ubi_repo_set_no_debug(): @pytest.fixture(name='test_ubiconfig') def fixture_test_ubiconfig(): - yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml") - + config = ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml") + config.version = '7.7' + yield config @pytest.fixture(name='executor') def fixture_executor(): @@ -68,6 +69,7 @@ def get_test_repo(**kwargs): kwargs.get('platform_full_version'), kwargs.get('distributors_ids_type_ids'), kwargs.get('ubi_population'), + kwargs.get('ubi_config_version'), ) diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 4d3582e..c434bf7 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -90,6 +90,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, content_sets=kwargs.get('content_sets', None), repo_ids=kwargs.get('repo_ids', None)) + self.ubiconfig_map = self._create_config_map() def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None): loader = ubiconfig.get_loader(ubiconfig_dir_or_url) @@ -132,10 +133,51 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids): return filtered_conf_list + def _create_config_map(self): + """Create a config map from self.ubiconfig_list, it has the form in: + { + "7.7": + { + "config_filename1": config1, + "config_filename2": config2, + ..., + }, + "8.1": + { + "config_filename1": config1, + ..., + }, + .... + } + """ + + config_map = {} + for config in self.ubiconfig_list: + config_map.setdefault(config.version, {})\ + .setdefault(config.filename, config) + + return config_map + def populate_ubi_repos(self): out_repos = set() - + used_content_sets = set() + # since repos are searched by content sets, same repo could be searched and ppopulated + # multiple times, to avoid that, cache the content sets already used and skip the config + # whose content sets are all in the cache for config in self.ubiconfig_list: + content_sets = [ + config.content_sets.rpm, + config.content_sets.srpm, + config.content_sets.debuginfo + ] + to_use = [cs for cs in content_sets if cs not in used_content_sets] + if to_use: + for cs in to_use: + used_content_sets.add(cs) + else: + _LOG.debug("Skipping %s, since its been used already") + continue + try: repo_pairs = self._get_input_and_output_repo_pairs(config) except RepoMissing: @@ -143,7 +185,13 @@ def populate_ubi_repos(self): continue for repo_set in repo_pairs: - UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run, + # get the right config file by ubi_config_version attr, if it's None, + # then it's not a mainline repo, use platform_full_version instead. + version = repo_set.out_repos.rpm.ubi_config_version \ + or repo_set.out_repos.rpm.platform_full_version + right_config = self.ubiconfig_map.get(version, {}).get(config.filename) + + UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run, self._executor).run_ubi_population() out_repos.update(repo_set.get_output_repo_ids()) diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 98e6958..0768908 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -90,6 +90,7 @@ def _search_repo(self, criteria): # Only UBI repos should have a ubi_population note, set to None for other platforms # If the UBI repo does not have this note, assume it is okay to populate ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None + ubi_config_version = notes.get('ubi_config_version', None) repos.append(Repo( repo_id=item['id'], arch=notes['arch'], @@ -97,6 +98,7 @@ def _search_repo(self, criteria): platform_full_version=notes['platform_full_version'], dist_ids_type_ids=dist_info, ubi_population=ubi_population, + ubi_config_version=ubi_config_version, )) return repos @@ -311,11 +313,12 @@ def publish_repo(self, repo): class Repo(object): def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids, - ubi_population): + ubi_population, ubi_config_version): self.repo_id = repo_id self.arch = arch self.content_set = content_set self.platform_full_version = platform_full_version + self.ubi_config_version = ubi_config_version self.distributors_ids_type_ids_tuples = dist_ids_type_ids self.ubi_population = ubi_population