From e658f0c61c897ba5e2bf5bf4588a55bd1a791d32 Mon Sep 17 00:00:00 2001 From: Hanjie Zhang <31371415+JayZ12138@users.noreply.github.com> Date: Thu, 24 Oct 2019 16:02:08 -0400 Subject: [PATCH] Use correct version of ubi config file for population (#140) * Use correct version of ubi config file for population This resloves #138 A new attr is added to Repo class and it will be populated while searching repos. After all config files are loaded, it now creates a config map, which helps find the right config file for the repo to populate. --- requirements.txt | 2 +- tests/data/{ => ubi7}/conf.yaml | 0 tests/data/{ => ubi8}/ubiconf_golang.yaml | 0 tests/data/ubi8/ubiconf_golang2.yaml | 52 ++++++ .../data/{ => ubi7}/associate-md.yaml | 0 .../data/{ => ubi7}/associate-pkg.yaml | 0 .../data/{ => ubi7}/clean-repos.yaml | 0 .../do-not-associate-excluded-pkg.yaml | 0 .../do-not-associate-pkg-if-in-exclude.yaml | 0 ...o-not-associate-pkg-if-not-in-include.yaml | 0 tests/integration/test_association.py | 2 +- tests/test_pulp.py | 6 + tests/test_ubipop.py | 150 +++++++++++++----- tests/test_utils.py | 15 +- ubipop/__init__.py | 86 +++++++++- ubipop/_pulp_client.py | 11 +- 16 files changed, 261 insertions(+), 63 deletions(-) rename tests/data/{ => ubi7}/conf.yaml (100%) rename tests/data/{ => ubi8}/ubiconf_golang.yaml (100%) create mode 100644 tests/data/ubi8/ubiconf_golang2.yaml rename tests/integration/data/{ => ubi7}/associate-md.yaml (100%) rename tests/integration/data/{ => ubi7}/associate-pkg.yaml (100%) rename tests/integration/data/{ => ubi7}/clean-repos.yaml (100%) rename tests/integration/data/{ => ubi7}/do-not-associate-excluded-pkg.yaml (100%) rename tests/integration/data/{ => ubi7}/do-not-associate-pkg-if-in-exclude.yaml (100%) rename tests/integration/data/{ => ubi7}/do-not-associate-pkg-if-not-in-include.yaml (100%) diff --git a/requirements.txt b/requirements.txt index 4edd1f6..6f893ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ requests more-executors -ubi-config +ubi-config>=2.2.0 rpm-py-installer diff --git a/tests/data/conf.yaml b/tests/data/ubi7/conf.yaml similarity index 100% rename from tests/data/conf.yaml rename to tests/data/ubi7/conf.yaml diff --git a/tests/data/ubiconf_golang.yaml b/tests/data/ubi8/ubiconf_golang.yaml similarity index 100% rename from tests/data/ubiconf_golang.yaml rename to tests/data/ubi8/ubiconf_golang.yaml diff --git a/tests/data/ubi8/ubiconf_golang2.yaml b/tests/data/ubi8/ubiconf_golang2.yaml new file mode 100644 index 0000000..7fa5fe7 --- /dev/null +++ b/tests/data/ubi8/ubiconf_golang2.yaml @@ -0,0 +1,52 @@ +content_sets: + rpm: + output: ubi-8-for-x86_64-appstream-rpms + input: rhel-8-for-x86_64-appstream-rpms + srpm: + output: ubi-8-for-x86_64-appstream-source-rpms + input: rhel-8-for-x86_64-appstream-source-rpms + debuginfo: + output: ubi-8-for-x86_64-appstream-debug-rpms + input: rhel-8-for-x86_64-appstream-debug-rpms + +arches: + - src + - x86_64 + +packages: + include: + - go-srpm-macros.* + - go-toolset.* + - golang.* + - golang-bin.* + - golang-docs.* + - golang-misc.* + - golang-race.* + - golang-src.* + - golang-tests.* + exclude: + - python36 + - kernel + - kernel-doc + - kmod-kvdo + - kernel.src + - kernel-debug-debuginfo + - kernel-debuginfo + - kernel-debuginfo-common-aarch64 + - kernel-debuginfo-common-ppc64le + - kernel-debuginfo-common-s390x + - kernel-debuginfo-common-x86_64 + - kernel-tools-debuginfo + - linux-firmware + - grub2 + - grub2-common + - grub2-tools + - grub2-tools-minimal + - grubby + - redhat-release-eula + - s390utils-base + +modules: + include: + - name: go-toolset + stream: rhel8 diff --git a/tests/integration/data/associate-md.yaml b/tests/integration/data/ubi7/associate-md.yaml similarity index 100% rename from tests/integration/data/associate-md.yaml rename to tests/integration/data/ubi7/associate-md.yaml diff --git a/tests/integration/data/associate-pkg.yaml b/tests/integration/data/ubi7/associate-pkg.yaml similarity index 100% rename from tests/integration/data/associate-pkg.yaml rename to tests/integration/data/ubi7/associate-pkg.yaml diff --git a/tests/integration/data/clean-repos.yaml b/tests/integration/data/ubi7/clean-repos.yaml similarity index 100% rename from tests/integration/data/clean-repos.yaml rename to tests/integration/data/ubi7/clean-repos.yaml diff --git a/tests/integration/data/do-not-associate-excluded-pkg.yaml b/tests/integration/data/ubi7/do-not-associate-excluded-pkg.yaml similarity index 100% rename from tests/integration/data/do-not-associate-excluded-pkg.yaml rename to tests/integration/data/ubi7/do-not-associate-excluded-pkg.yaml diff --git a/tests/integration/data/do-not-associate-pkg-if-in-exclude.yaml b/tests/integration/data/ubi7/do-not-associate-pkg-if-in-exclude.yaml similarity index 100% rename from tests/integration/data/do-not-associate-pkg-if-in-exclude.yaml rename to tests/integration/data/ubi7/do-not-associate-pkg-if-in-exclude.yaml diff --git a/tests/integration/data/do-not-associate-pkg-if-not-in-include.yaml b/tests/integration/data/ubi7/do-not-associate-pkg-if-not-in-include.yaml similarity index 100% rename from tests/integration/data/do-not-associate-pkg-if-not-in-include.yaml rename to tests/integration/data/ubi7/do-not-associate-pkg-if-not-in-include.yaml diff --git a/tests/integration/test_association.py b/tests/integration/test_association.py index 20a9827..02253a8 100644 --- a/tests/integration/test_association.py +++ b/tests/integration/test_association.py @@ -19,7 +19,7 @@ PULP_SECURE = os.getenv('TEST_PULP_SECURE', '0').lower() in ['true', 'yes', '1'] PULP_RPM_REPO_PREFIX = '/pulp/rpmrepos/' -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data') +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), './data/ubi7') INTEGRATION_NOT_SETUP = PULP_HOSTNAME is None diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 8522a5a..728e319 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -43,6 +43,8 @@ def fixture_search_repo_response(): "content_set": "test_repo-source-rpms", "platform": "ubi", "platform_full_version": "7", + "platform_major_version": "7", + "ubi_config_version": "7", }, "distributors": [{ "id": "dist_id", @@ -58,6 +60,8 @@ def fixture_mock_repo(): arch="x86_64", content_set="test_repo-source-rpms", platform_full_version="7", + platform_major_version="7", + ubi_config_version="7", dist_ids_type_ids=[ ("dist_id_1", "dist_type_id_1"), ("dist_id_2", "dist_type_id_2"), @@ -184,6 +188,7 @@ def test_search_repo_by_cs(mock_pulp, mock_search_repos): assert repo.arch == "x86_64" assert repo.content_set == "test_repo-source-rpms" assert repo.platform_full_version == "7" + assert repo.platform_major_version == "7" assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id") # ubi_population note was unset, so it should default to True assert repo.ubi_population is True @@ -199,6 +204,7 @@ def test_search_repo_by_id(mock_pulp, mock_search_repos): assert repo.arch == "x86_64" assert repo.content_set == "test_repo-source-rpms" assert repo.platform_full_version == "7" + assert repo.ubi_config_version == "7" assert repo.distributors_ids_type_ids_tuples[0] == ("dist_id", "d_type_id") # ubi_population note was unset, so it should default to True assert repo.ubi_population is True diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 0ea6e23..3ae3791 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -12,7 +12,7 @@ from mock import MagicMock, patch, call from more_executors import Executors -from ubipop import UbiPopulateRunner, UbiRepoSet, RepoSet, UbiPopulate +from ubipop import UbiPopulateRunner, UbiRepoSet, RepoSet, UbiPopulate, ConfigMissing from ubipop._pulp_client import Module, ModuleDefaults, Package, Repo from ubipop._utils import ( AssociateActionModules, @@ -27,12 +27,18 @@ @pytest.fixture(name='ubi_repo_set') def fixture_ubi_repo_set(): - yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms")], - [get_test_repo(repo_id="foo-source")], - [get_test_repo(repo_id="foo-debug")]), - RepoSet(get_test_repo(repo_id="ubi-foo-rpms"), - get_test_repo(repo_id="ubi-foo-source"), - get_test_repo(repo_id="ubi-foo-debug"))) + yield UbiRepoSet(RepoSet([get_test_repo(repo_id="foo-rpms", + ubi_config_version="7")], + [get_test_repo(repo_id="foo-source", + ubi_config_version="7")], + [get_test_repo(repo_id="foo-debug", + ubi_config_version="7")]), + RepoSet(get_test_repo(repo_id="ubi-foo-rpms", + ubi_config_version="7"), + get_test_repo(repo_id="ubi-foo-source", + ubi_config_version="7"), + get_test_repo(repo_id="ubi-foo-debug", + ubi_config_version="7"))) @pytest.fixture(name='ubi_repo_set_no_debug') @@ -47,7 +53,7 @@ def fixture_ubi_repo_set_no_debug(): @pytest.fixture(name='test_ubiconfig') def fixture_test_ubiconfig(): - yield ubiconfig.get_loader(TEST_DATA_DIR).load("conf.yaml") + yield ubiconfig.get_loader(TEST_DATA_DIR).load("ubi7/conf.yaml") @pytest.fixture(name='executor') @@ -66,9 +72,11 @@ def get_test_repo(**kwargs): kwargs.get('arch'), kwargs.get('content_set'), kwargs.get('platform_full_version'), + kwargs.get('platform_major_version'), kwargs.get('distributors_ids_type_ids'), kwargs.get('ubi_population'), kwargs.get('population_sources'), + kwargs.get('ubi_config_version'), ) @@ -118,21 +126,49 @@ def test_get_output_repo_ids_no_debug(ubi_repo_set_no_debug): def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog): # Don't actually query Pulp for repos mocked_search_repo_by_cs.side_effect = [ + # Output repos - rhel-7-server + [get_test_repo( + repo_id="ubi-7-server-rpms__7_DOT_2__x86_64", + content_set="ubi-7-server-rpms", + ubi_population=False, + platform_full_version="7.2", + ), ], + [get_test_repo( + repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64", + content_set="ubi-7-server-source-rpms", + ubi_population=True, + # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo + # for skipping whole repo triplet + platform_full_version="7.2", + ), ], + [get_test_repo( + repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64", + content_set="ubi-7-server-debuginfo-rpms", + ubi_population=False, + platform_full_version="7.2" + ), ], + # Output repos - rhel-8-for-x86_64-appstream [get_test_repo( repo_id="ubi-8-for-x86_64-appstream-rpms", content_set="ubi-8-for-x86_64-appstream-rpms", - ubi_population=True + ubi_population=True, + ubi_config_version="8", + platform_full_version="8" ), ], [get_test_repo( repo_id="ubi-8-for-x86_64-appstream-source-rpms", content_set="ubi-8-for-x86_64-appstream-source-rpms", - ubi_population=True + ubi_population=True, + ubi_config_version="8", + platform_full_version="8" ), ], [get_test_repo( repo_id="ubi-8-for-x86_64-appstream-debug-rpms", content_set="ubi-8-for-x86_64-appstream-debug-rpms", - ubi_population=True + ubi_population=True, + ubi_config_version="8", + platform_full_version="8" ), ], # Input repos - rhel-8-for-x86_64-appstream @@ -148,51 +184,71 @@ def test_skip_outdated_dot_repos(mocked_search_repo_by_cs, mocked_ubipop_runner, repo_id="rhel-8-for-x86_64-appstream-debug-rpms", content_set="rhel-8-for-x86_64-appstream-debug-rpms", ), ], + ] - # Output repos - rhel-7-server + # Attempt to populate both invalid and valid repo sets + ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR) + ubipop.populate_ubi_repos() + + # Should've only run once + assert mocked_ubipop_runner.call_count == 1 + # For ubi-8 + assert "ubi-8-for-x86_64-appstream-rpms" not in caplog.text + # Not for ubi-7-server + assert "ubi-7-server-rpms__7_DOT_2__x86_64" in caplog.text + # Used content sets won't be used again + assert "Skipping ubiconf_golang2.yaml, since it's been used already" in caplog.text + + +@patch("ubipop.UbiPopulateRunner") +@patch("ubipop._pulp_client.Pulp.search_repo_by_cs") +def test_raise_config_missing(mocked_search_repo_by_cs, mocked_ubipop_runner, caplog): + mocked_search_repo_by_cs.side_effect = [ + # Output repos - rhel-7-for-x86_64-appstream [get_test_repo( - repo_id="ubi-7-server-rpms__7_DOT_2__x86_64", - content_set="ubi-7-server-rpms", - ubi_population=False + repo_id="ubi-7-for-x86_64-appstream-rpms", + content_set="ubi-7-for-x86_64-appstream-rpms", + ubi_population=True, + platform_full_version="7", + platform_major_version="7" ), ], [get_test_repo( - repo_id="ubi-7-server-source-rpms__7_DOT_2__x86_64", - content_set="ubi-7-server-source-rpms", - ubi_population=True - # doesn't matter here, it sufficient to have ubi_population==False at rpm binary repo - # for skipping whole repo triplet + repo_id="ubi-7-for-x86_64-appstream-source-rpms", + content_set="ubi-7-for-x86_64-appstream-source-rpms", + ubi_population=True, + platform_full_version="7", + platform_major_version="7" ), ], [get_test_repo( - repo_id="ubi-7-server-debuginfo-rpms__7_DOT_2__x86_64", - content_set="ubi-7-server-debuginfo-rpms", - ubi_population=False + repo_id="ubi-7-for-x86_64-appstream-debug-rpms", + content_set="ubi-7-for-x86_64-appstream-debug-rpms", + ubi_population=True, + platform_full_version="7", + platform_major_version="7" ), ], - # Input repos - rhel-7-server + # Input repos - rhel-7-for-x86_64-appstream [get_test_repo( - repo_id="rhel-7-server-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-rpms", + repo_id="rhel-7-for-x86_64-appstream-rpms", + content_set="rhel-7-for-x86_64-appstream-rpms", ), ], [get_test_repo( - repo_id="rhel-7-server-source-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-source-rpms", + repo_id="rhel-7-for-x86_64-appstream-source-rpms", + content_set="rhel-7-for-x86_64-appstream-source-rpms", ), ], [get_test_repo( - repo_id="rhel-7-server-debuginfo-rpms__7_DOT_2__x86_64", - content_set="rhel-7-server-debuginfo-rpms", + repo_id="rhel-7-for-x86_64-appstream-debug-rpms", + content_set="rhel-7-for-x86_64-appstream-debug-rpms", ), ], ] + config_path = os.path.join(TEST_DATA_DIR, 'ubi8') + ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=config_path) + with pytest.raises(ConfigMissing): + ubipop.populate_ubi_repos() - # Attempt to populate both invalid and valid repo sets - ubipop = UbiPopulate("foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=TEST_DATA_DIR) - ubipop.populate_ubi_repos() + assert mocked_ubipop_runner.call_count == 0 - # Should've only run once - assert mocked_ubipop_runner.call_count == 1 - # For ubi-8 - assert "ubi-8-for-x86_64-appstream-rpms" not in caplog.text - # Not for ubi-7-server - assert "ubi-7-server-rpms__7_DOT_2__x86_64" in caplog.text + assert "Config file ubiconf_golang.yaml missing from 7 and default 7 branches" in caplog.text @patch("ubipop._pulp_client.Pulp.search_repo_by_id") @@ -522,7 +578,7 @@ def test_match_modules(mock_ubipop_runner): def test_match_modules_without_profile(ubi_repo_set, executor): - test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubiconf_golang.yaml') + test_ubiconf = ubiconfig.get_loader(TEST_DATA_DIR).load('ubi8/ubiconf_golang.yaml') mocked_ubipop_runner = UbiPopulateRunner( MagicMock(), ubi_repo_set, @@ -771,7 +827,10 @@ def test_ubi_repo_set(rhel_repo_set, ubi_repo_set, fail, caplog): @pytest.fixture(name='mocked_ubiconfig_load') def fixture_mocked_ubiconfig_load(): with patch('ubiconfig.get_loader') as get_loader: - get_loader.return_value.load.return_value = "test" + m = MagicMock() + m.file_name = "test" + m.version = "7.7" + get_loader.return_value.load.return_value = m yield get_loader @@ -779,7 +838,7 @@ def test_ubipopulate_load_ubiconfig(mocked_ubiconfig_load): # pylint: disable=unused-argument ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False, ['cfg.yaml']) assert len(ubipop.ubiconfig_list) == 1 - assert ubipop.ubiconfig_list[0] == "test" + assert ubipop.ubiconfig_list[0].file_name == "test" def test_load_ubiconfig_by_content_set_labels(): @@ -805,7 +864,10 @@ def test_load_ubiconfig_by_repo_ids(mocked_search_repo_by_id): @pytest.fixture(name='mocked_ubiconfig_load_all') def fixture_mocked_ubiconfig_load_all(): with patch('ubiconfig.get_loader') as get_loader: - get_loader.return_value.load_all.return_value = ["test"] + m = MagicMock() + m.file_name = "test" + m.version = "7" + get_loader.return_value.load_all.return_value = [m] yield get_loader @@ -813,7 +875,7 @@ def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all): # pylint: disable=unused-argument ubipop = UbiPopulate("foo.pulp.com", ('foo', 'foo'), False) assert len(ubipop.ubiconfig_list) == 1 - assert ubipop.ubiconfig_list[0] == "test" + assert ubipop.ubiconfig_list[0].file_name == "test" def test_create_srpms_output_set(mock_ubipop_runner): diff --git a/tests/test_utils.py b/tests/test_utils.py index ad271be..712cd09 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,15 +16,15 @@ def test_raise_not_implemented_pulp_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None, None) + repo = Repo("test", "1", "test-rpms", "2", "2", "2", None, None, None) action = PulpAction(units, repo) pytest.raises(NotImplementedError, action.get_actions, None) def test_raise_not_implemented_associate_action(): units = ["unit1", "unit2"] - repo = Repo("test", "1", "test-rpms", "2", None, None, None) - src_repo = Repo("test", "1", "test-rpms", "2", None, None, None) + repo = Repo("test", "1", "test-rpms", "2", "2", "2", None, None, None) + src_repo = Repo("test", "1", "test-rpms", "2", "2", "2", None, None, None) action = AssociateAction(units, repo, src_repo) pytest.raises(NotImplementedError, action.get_actions, None) @@ -40,12 +40,11 @@ def test_get_action_associate(klass, method): mocked_unit_2 = MagicMock() mocked_unit_2.associate_source_repo_id = "test_src_2" units = [mocked_unit_1, mocked_unit_2] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None) - src_repos = [Repo("test_src_1", "1", "test_src-rpms", "2", None, None, None), - Repo("test_src_2", "1", "test_src-rpms", "2", None, None, None)] + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", "2", None, None, None) + src_repos = [Repo("test_src_1", "1", "test_src-rpms", "2", "2", "2", None, None, None), + Repo("test_src_2", "1", "test_src-rpms", "2", "2", "2", None, None, None)] action = klass(units, dst_repo, src_repos) actions = action.get_actions(MagicMock()) - for action in actions: associate_action, src_repo_current, dst_repo_current, current_units = action assert "mock." + method in str(associate_action) @@ -63,7 +62,7 @@ def test_get_action_associate(klass, method): ]) def test_get_action_unassociate(klass, method): units = ["unit1", "unit2"] - dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", None, None, None) + dst_repo = Repo("test_dst", "1", "test_dst-rpms", "2", "2", "2", None, None, None) action = klass(units, dst_repo) associate_action, dst_repo_current, current_units = action.get_actions(MagicMock())[0] diff --git a/ubipop/__init__.py b/ubipop/__init__.py index cf68385..44f68e4 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -26,6 +26,10 @@ class RepoMissing(Exception): pass +class ConfigMissing(Exception): + pass + + RepoSet = namedtuple('RepoSet', ['rpm', 'source', 'debug']) @@ -90,6 +94,7 @@ def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=No self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, content_sets=kwargs.get('content_sets', None), repo_ids=kwargs.get('repo_ids', None)) + self.ubiconfig_map = self._create_config_map() def _load_ubiconfig(self, filenames, ubiconfig_dir_or_url, content_sets=None, repo_ids=None): loader = ubiconfig.get_loader(ubiconfig_dir_or_url) @@ -122,9 +127,9 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids): for conf in config_list: for label in [ - conf.content_sets.rpm.input, conf.content_sets.rpm.output, - conf.content_sets.srpm.input, conf.content_sets.srpm.output, - conf.content_sets.debuginfo.input, conf.content_sets.debuginfo.output, + conf.content_sets.rpm.input, conf.content_sets.rpm.output, + conf.content_sets.srpm.input, conf.content_sets.srpm.output, + conf.content_sets.debuginfo.input, conf.content_sets.debuginfo.output, ]: if label in content_sets: filtered_conf_list.append(conf) @@ -132,10 +137,53 @@ def _filter_ubi_conf_list(self, config_list, content_sets, repo_ids): return filtered_conf_list + def _create_config_map(self): + """Create a config map from self.ubiconfig_list, it has the form in: + { + "7.7": + { + "config_filename1": config1, + "config_filename2": config2, + ..., + }, + "8.1": + { + "config_filename1": config1, + ..., + }, + .... + } + """ + + config_map = {} + for config in self.ubiconfig_list: + config_map.setdefault(config.version, {})\ + .setdefault(config.file_name, config) + + return config_map + def populate_ubi_repos(self): out_repos = set() + used_content_sets = set() + # since repos are searched by content sets, same repo could be searched and populated + # multiple times, to avoid that, cache the content sets already used and skip the config + # whose content sets are all in the cache + + for config in sorted(self.ubiconfig_list, key=str): + content_sets = [ + config.content_sets.rpm.output, + config.content_sets.srpm.output, + config.content_sets.debuginfo.output, + ] + + to_use = [cs for cs in content_sets if cs not in used_content_sets] + if to_use: + for cs in to_use: + used_content_sets.add(cs) + else: + _LOG.debug("Skipping %s, since it's been used already", config.file_name) + continue - for config in self.ubiconfig_list: try: repo_pairs = self._get_ubi_repo_sets(config) @@ -144,7 +192,33 @@ def populate_ubi_repos(self): continue for repo_set in repo_pairs: - UbiPopulateRunner(self.pulp, repo_set, config, self.dry_run, + ubi_config_version = repo_set.out_repos.rpm.ubi_config_version + platform_full_version = repo_set.out_repos.rpm.platform_full_version + platform_major_version = repo_set.out_repos.rpm.platform_major_version + # get the right config file by ubi_config_version attr, if it's None, + # then it's not a mainline repo, use platform_full_version instead. + # config file could also be missing for specific version, then the + # default config file will be used. + version = ubi_config_version or platform_full_version + right_config = self.ubiconfig_map\ + .get(str(version), {})\ + .get(config.file_name)\ + or self.ubiconfig_map\ + .get(str(platform_major_version), {})\ + .get(config.file_name) + + # if config file is missing from wanted version, as well as default + # branch, raise exception + if not right_config: + _LOG.error( + 'Config file %s missing from %s and default %s branches', + config.file_name, + version, + platform_full_version, + ) + raise ConfigMissing() + + UbiPopulateRunner(self.pulp, repo_set, right_config, self.dry_run, self._executor).run_ubi_population() out_repos.update(repo_set.get_output_repo_ids()) @@ -158,7 +232,7 @@ def _get_ubi_repo_sets(self, ubi_config_item): """ Searches for ubi repository triplet (binary rpm, srpm, debug) for one ubi config item and tries to determine their population sources - (input repositories). Returns list UbiRepoSet objects that provides + (input repositories). Returns list UbiRepoSet objects that provides input and output repositories that are used for population process. """ rpm_repos_ft = self._executor.submit(self.pulp.search_repo_by_cs, diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 6c1e6e6..22832a5 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -90,14 +90,17 @@ def _search_repo(self, criteria): # Only UBI repos should have a ubi_population note, set to None for other platforms # If the UBI repo does not have this note, assume it is okay to populate ubi_population = notes.get('ubi_population', True) if notes['platform'] == 'ubi' else None + ubi_config_version = notes.get('ubi_config_version', None) repos.append(Repo( repo_id=item['id'], arch=notes['arch'], content_set=notes['content_set'], platform_full_version=notes['platform_full_version'], + platform_major_version=notes['platform_major_version'], dist_ids_type_ids=dist_info, ubi_population=ubi_population, - population_sources=notes.get('population_sources') + population_sources=notes.get('population_sources'), + ubi_config_version=ubi_config_version, )) return repos @@ -312,12 +315,14 @@ def publish_repo(self, repo): class Repo(object): - def __init__(self, repo_id, arch, content_set, platform_full_version, dist_ids_type_ids, - ubi_population, population_sources): + def __init__(self, repo_id, arch, content_set, platform_full_version, platform_major_version, + dist_ids_type_ids, ubi_population, population_sources, ubi_config_version): self.repo_id = repo_id self.arch = arch self.content_set = content_set self.platform_full_version = platform_full_version + self.platform_major_version = platform_major_version + self.ubi_config_version = ubi_config_version self.distributors_ids_type_ids_tuples = dist_ids_type_ids self.ubi_population = ubi_population self.population_sources = population_sources