From b0a9f9ec42ee7c64fb628c5cbb0acbe7e67092aa Mon Sep 17 00:00:00 2001 From: Robert Bikar Date: Tue, 24 Aug 2021 17:13:23 +0200 Subject: [PATCH] RPMs matching re-implemented [RHELDST-6392] TODO --- tests/test_matcher.py | 377 +++++++++++++++++++++++++++++- tests/test_ubipop.py | 529 ++++++++++++++---------------------------- tests/test_utils.py | 41 +++- ubipop/__init__.py | 228 +----------------- ubipop/_matcher.py | 254 ++++++++++++++++++-- ubipop/_utils.py | 29 +++ 6 files changed, 855 insertions(+), 603 deletions(-) diff --git a/tests/test_matcher.py b/tests/test_matcher.py index 761b45b..88f1e1c 100644 --- a/tests/test_matcher.py +++ b/tests/test_matcher.py @@ -1,6 +1,6 @@ +from operator import attrgetter import pytest -from operator import attrgetter from more_executors.futures import f_proxy, f_return from pubtools.pulplib import ( RpmUnit, @@ -11,8 +11,15 @@ ) from ubiconfig import UbiConfig -from ubipop._matcher import UbiUnit, Matcher, flatten_list_of_sets, ModularMatcher +from ubipop._matcher import ( + UbiUnit, + Matcher, + flatten_list_of_sets, + ModularMatcher, + RpmMatcher, +) from ubipop import RepoSet +from ubipop._utils import vercmp_sort @pytest.fixture(name="pulp") @@ -23,6 +30,7 @@ def fake_pulp(): @pytest.fixture(name="ubi_config") def fake_ubi_config(): config_dict = { + "arches": ["src"], "modules": { "include": [ { @@ -32,7 +40,14 @@ def fake_ubi_config(): } ] }, - "packages": {}, + "packages": { + "include": ["test.*", "something_else.src"], + "exclude": [ + "excluded_with_globbing*", + "excluded_package.*", + "excluded_with_arch.src", + ], + }, "content_sets": {}, } yield UbiConfig.load_from_dict(config_dict, "fake/config.yaml") @@ -504,7 +519,7 @@ def test_get_modular_srpms_criteria(ubi_config): matcher.binary_rpms = f_proxy(f_return(set([unit_1]))) matcher.debug_rpms = f_proxy(f_return(set([unit_2]))) - criteria = matcher._get_modular_srpms_criteria() + criteria = matcher._get_srpms_criteria() # there should be 1 criteria created assert len(criteria) == 2 # it should be instance of Criteria @@ -634,6 +649,338 @@ def test_modular_matcher_run(pulp, ubi_config): assert rpm.associate_source_repo_id == "source_repo" +def test_parse_blacklist_config(ubi_config): + matcher = RpmMatcher(None, ubi_config) + blacklist_parsed = sorted(matcher._parse_blacklist_config()) + + assert len(blacklist_parsed) == 3 + + expected_blacklist_parsed = sorted( + [ + ("excluded_package", False, None), + ("excluded_with_globbing", True, None), + ("excluded_with_arch", False, "src"), + ] + ) + + assert blacklist_parsed == expected_blacklist_parsed + + +def test_keep_n_latest_rpms(): + + """Test keeping only the latest version of modulemd""" + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + + unit_2 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + + matcher = RpmMatcher(None, None) + rpms = [unit_1, unit_2] + rpms.sort(key=vercmp_sort()) + matcher._keep_n_latest_rpms(rpms) + + # there should only one modulemd + assert len(rpms) == 1 + # with the highest number of version + assert rpms[0].version == "11" + + +def test_keep_n_latest_rpms_multiple_arches(): + + """Test keeping only the latest version of modulemd""" + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + + unit_2 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + unit_3 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="i686", + ), + None, + ) + + matcher = RpmMatcher(None, None) + rpms = [unit_1, unit_2, unit_3] + rpms.sort(key=vercmp_sort()) + matcher._keep_n_latest_rpms(rpms) + + # there should only one modulemd + assert len(rpms) == 2 + # with the highest number of version + assert rpms[0].version == "11" + assert rpms[0].arch == "x86_64" + + # with the highest number of version + assert rpms[1].version == "10" + assert rpms[1].arch == "i686" + + +def test_get_rpm_output_set(ubi_config): + # blacklist is applied + modular_rpms striiped + ### TODO co pouziti futures, aby to stymovalo + matcher = RpmMatcher(None, ubi_config) + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + + unit_2 = UbiUnit( + RpmUnit( + name="excluded_with_globbing123456789", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + unit_3 = UbiUnit( + RpmUnit( + name="excluded_package", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + unit_4 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + unit_5 = UbiUnit( + RpmUnit( + name="modular_package", + version="10", + release="20", + arch="x86_64", + filename="modular_package.rpm", + ), + None, + ) + + modular_pkgs_filenames = f_proxy(f_return(set(["modular_package.rpm"]))) + rpms = [unit_1, unit_2, unit_3, unit_4, unit_5] + output = matcher._get_rpm_output_set(rpms, modular_pkgs_filenames) + + assert len(output) == 1 + assert output[0].name == "test" + assert output[0].version == "11" + + +def test_get_pkgs_from_all_modules(pulp): + repo = YumRepository( + id="test_repo_1", + ) + repo.__dict__["_client"] = pulp.client + unit_1 = ModulemdUnit( + name="test", + stream="10", + version=100, + context="abcdef", + arch="x86_64", + artifacts=[ + "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src", + "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64", + ], + ) + unit_2 = ModulemdUnit( + name="test", + stream="20", + version=100, + context="abcdef", + arch="x86_64", + artifacts=[ + "perl-version-7:1.99.24-441.module+el8.4.0+9911+7f269185.src", + "perl-version-7:1.99.24-441.module+el8.4.0+9911+7f269185.x86_64", + ], + ) + + pulp.insert_repository(repo) + pulp.insert_units(repo, [unit_1, unit_2]) + repos_set = RepoSet(rpm=[repo], debug=[], source=[]) + matcher = RpmMatcher(repos_set, None) + modular_filenames = matcher._get_pkgs_from_all_modules().result() + # there should be be only one unit in the result set according to criteria + + expected_filenames = set( + [ + "perl-version-0.99.24-441.module+el8.3.0+6718+7f269185.src.rpm", + "perl-version-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm", + "perl-version-1.99.24-441.module+el8.4.0+9911+7f269185.src.rpm", + "perl-version-1.99.24-441.module+el8.4.0+9911+7f269185.x86_64.rpm", + ] + ) + + assert len(modular_filenames) == 4 + assert modular_filenames == expected_filenames + + +def test_get_rpms_criteria(ubi_config): + matcher = RpmMatcher(None, ubi_config) + criteria = matcher._get_rpms_criteria() + # there should be 1 criterium created based on ubi config + assert len(criteria) == 1 + # it should be instance of Criteria + for crit in criteria: + assert isinstance(crit, Criteria) + # let's not test internal structure of criteria, that's responsibility of pulplib + + +def test_rpm_matcher_run(pulp, ubi_config): + """Test run() method which asynchronously creates criteria for queries to pulp + and exectues those query. Finally it sets up public attrs of the ModularMatcher + object that can be used in ubipop""" + + repo_1 = YumRepository( + id="binary_repo", + ) + repo_1.__dict__["_client"] = pulp.client + + repo_2 = YumRepository( + id="debug_repo", + ) + repo_2.__dict__["_client"] = pulp.client + repo_3 = YumRepository( + id="source_repo", + ) + repo_3.__dict__["_client"] = pulp.client + + # binary + unit_1 = RpmUnit( + name="test", + version="1.0", + release="1", + arch="x86_64", + filename="test-1.0-1.x86_64.rpm", + sourcerpm="test-1.0-1.src.rpm", + ) + # debug + unit_2 = RpmUnit( + name="test", + version="1.0", + release="1", + arch="x86_64", + filename="test-debug-1.0-1.x86_64.rpm", + ) + # source + unit_3 = RpmUnit( + name="test-src", + version="1.0", + release="1", + arch="src", + filename="test-1.0-1.src.rpm", + content_type_id="srpm", + ) + + # modular - will be skipped + unit_4 = RpmUnit( + name="test", + version="100.0", + release="1", + arch="x86_64", + filename="test-100.0-1.x86_64.rpm", + sourcerpm="test-100.0-1.src.rpm", + ) + # blacklisted - will be also skipped + unit_5 = RpmUnit( + name="excluded_package", + version="1.0", + release="1", + arch="x86_64", + filename="excluded_package-1.0-1.x86_64.rpm", + sourcerpm="excluded_package-1.0-1.src.rpm", + ) + # non-matching - not in output + unit_6 = RpmUnit( + name="no_match", + version="1.0", + release="1", + arch="x86_64", + filename="no_match-1.0-1.x86_64.rpm", + sourcerpm="no_match-1.0-1.src.rpm", + ) + + modulemd = ModulemdUnit( + name="fake_name", + stream="fake_stream", + version=100, + context="abcd", + arch="x86_64", + artifacts=[ + "test-0:100.0-1.x86_64", + ], + ) + pulp.insert_repository(repo_1) + pulp.insert_repository(repo_2) + pulp.insert_repository(repo_3) + pulp.insert_units(repo_1, [unit_1, modulemd, unit_4, unit_5, unit_6]) + pulp.insert_units(repo_2, [unit_2]) + pulp.insert_units(repo_3, [unit_3]) + + repos_set = RepoSet(rpm=[repo_1], debug=[repo_2], source=[repo_3]) + matcher = RpmMatcher(repos_set, ubi_config) + matcher.run() + # each public attribute is properly set with one unit + assert len(matcher.binary_rpms) == 1 + assert len(matcher.debug_rpms) == 1 + assert len(matcher.source_rpms) == 1 + + # each unit is properly queried + rpm = matcher.binary_rpms.pop() + assert rpm.filename == "test-1.0-1.x86_64.rpm" + assert rpm.associate_source_repo_id == "binary_repo" + + rpm = matcher.debug_rpms.pop() + assert rpm.filename == "test-debug-1.0-1.x86_64.rpm" + assert rpm.associate_source_repo_id == "debug_repo" + + rpm = matcher.source_rpms.pop() + assert rpm.filename == "test-1.0-1.src.rpm" + assert rpm.associate_source_repo_id == "source_repo" + + def test_flatten_list_of_sets(): """Test helper function that flattens list of sets into one set""" set_1 = set([1, 2, 3]) @@ -642,3 +989,25 @@ def test_flatten_list_of_sets(): new_set = flatten_list_of_sets([set_1, set_2]).result() assert new_set == expected_set + + +""" +def test_get_blacklisted_packages_match_arch(mock_ubipop_runner): + pkg_name = "foo-arch-test" + test_pkg_list = [ + get_test_pkg( + name=pkg_name, + filename="{name}-3.0.6-4.el7.noarch.rpm".format(name=pkg_name), + ), + get_test_pkg( + name=pkg_name, + filename="{name}-3.0.6-4.el7.x86_64.rpm".format(name=pkg_name), + ), + ] + + blacklist = mock_ubipop_runner.get_blacklisted_packages(test_pkg_list) + + assert len(blacklist) == 1 + assert blacklist[0].name == pkg_name + assert "x86_64" in blacklist[0].filename +""" diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 4d45d97..f955612 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -354,45 +354,6 @@ def test_get_ubi_repo_sets(get_debug_repository, get_source_repository): assert output_repos.debug.id == "ubi_debug" -def test_get_blacklisted_packages_match_name_glob(mock_ubipop_runner): - pkg_name = "foo-pkg" - test_pkg_list = [ - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.noarch.rpm".format(name=pkg_name), - ), - get_test_pkg( - name="no-match-foo-pkg", - filename="no-match-foo-pkg-3.0.6-4.el7.noarch.rpm", - ), - ] - - blacklist = mock_ubipop_runner.get_blacklisted_packages(test_pkg_list) - - assert len(blacklist) == 1 - assert blacklist[0].name == pkg_name - - -def test_get_blacklisted_packages_match_arch(mock_ubipop_runner): - pkg_name = "foo-arch-test" - test_pkg_list = [ - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.noarch.rpm".format(name=pkg_name), - ), - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.x86_64.rpm".format(name=pkg_name), - ), - ] - - blacklist = mock_ubipop_runner.get_blacklisted_packages(test_pkg_list) - - assert len(blacklist) == 1 - assert blacklist[0].name == pkg_name - assert "x86_64" in blacklist[0].filename - - def _get_search_rpms_side_effect(package_name_or_filename_or_list, debug_only=False): def _f(*args, **kwargs): if debug_only and "debug" not in args[0].id: @@ -424,62 +385,6 @@ def _f(*args, **kwargs): return _f -def test_match_binary_rpms_skip_modular(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=["foo-2-pkg-0:7.3.6-1.el8+1944+b6c8e16f.noarch", "foo-pkg0:"], - ), - ] - - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - "foo-pkg", - ) - - mock_ubipop_runner._match_binary_rpms() # pylint: disable=protected-access - # there are no packages stored, modular ones were skipped even if they match - # by ubi_config - assert len(mock_ubipop_runner.repos.packages) == 0 - - -def test_match_binary_rpms(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=[ - "foo-2-pkg-0:7.3.6-1.el8+1944+b6c8e16f.noarch", - ], - ), - ] - - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - "foo-pkg", - ) - - mock_ubipop_runner._match_binary_rpms() # pylint: disable=protected-access - - current_pkg = mock_ubipop_runner.repos.packages["foo-pkg"][0] - assert len(mock_ubipop_runner.repos.packages) == 1 - assert current_pkg.name == "foo-pkg" - assert current_pkg.filename == "foo-pkg.rpm" - # we skip handling modular packages in _match_binary_rpms() method completely - # by default is_modular attr is set to False - assert current_pkg.is_modular is False - - -def test_match_debug_rpms(mock_ubipop_runner): - package_name = "foo-pkg-debuginfo" - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - package_name - ) - mock_ubipop_runner._match_debug_rpms() # pylint: disable=protected-access - - assert len(mock_ubipop_runner.repos.debug_rpms) == 1 - assert mock_ubipop_runner.repos.debug_rpms[package_name][0].name == package_name - - def test_match_module_defaults(mock_ubipop_runner): unit = UbiUnit( ModulemdUnit( @@ -526,62 +431,6 @@ def test_diff_modules(mock_ubipop_runner): assert diff[0].name == "1" -def test_keep_n_newest_packages(mock_ubipop_runner): - packages = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.8-1.el8+1944+b6c8e16f.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - packages.sort() - mock_ubipop_runner.keep_n_latest_packages(packages) - - assert len(packages) == 1 - assert "7.3.8" in packages[0].filename - - -def test_keep_n_newest_packages_multi_arch(mock_ubipop_runner): - packages = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.x86_64.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.i686.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.5-1.i686.rpm", - ), - ] - - packages.sort() - mock_ubipop_runner.keep_n_latest_packages(packages) - assert len(packages) == 3 - - arches_expected = ["noarch", "x86_64", "i686"] - arches_current = [] - - for pkg in packages: - _, _, _, _, arch = split_filename(pkg.filename) - arches_current.append(arch) - - assert sorted(arches_current) == sorted(arches_expected) - - @pytest.mark.parametrize( "rhel_repo_set, ubi_repo_set, fail", [ @@ -686,66 +535,6 @@ def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all): assert ubipop.ubiconfig_list[0].file_name == "test" -def test_create_srpms_output_set(mock_ubipop_runner): - mock_ubipop_runner.repos.packages["foo"] = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.src.rpm", - src_repo_id="foo-rpms", - ), - # blacklisted - get_test_pkg( - name="kernel", - filename="kernel-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="kernel.src.rpm", - src_repo_id="foo-rpms", - ), - # blacklisted but referenced in some module - get_test_pkg( - name="foo-pkg", - filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.src.rpm", - is_modular=True, - src_repo_id="foo-rpms", - ), - ] - - mock_ubipop_runner._create_srpms_output_set() # pylint: disable=protected-access - - out_srpms = mock_ubipop_runner.repos.source_rpms - assert len(out_srpms) == 2 - assert "tomcatjss" and "foo-pkg" in out_srpms - assert ( - out_srpms["tomcatjss"][0].filename - == "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.src.rpm" - ) - assert ( - out_srpms["foo-pkg"][0].filename == "foo-pkg-7.3.6-1.el8+1944+b6c8e16f.src.rpm" - ) - - -def test_create_srpms_output_set_missings_srpm_reference( - capsys, set_logging, mock_ubipop_runner -): - set_logging.addHandler(logging.StreamHandler(sys.stdout)) - mock_ubipop_runner.repos.packages["foo"] = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._create_srpms_output_set() # pylint: disable=protected-access - - out_srpms = mock_ubipop_runner.repos.source_rpms - assert len(out_srpms) == 0 - out, err = capsys.readouterr() - - assert err == "" - assert out.strip() == "Package tomcatjss doesn't reference its source rpm" - - @pytest.fixture(name="mock_get_repo_pairs") def fixture_mock_get_repo_pairs(ubi_repo_set): with patch("ubipop.UbiPopulate._get_ubi_repo_sets") as get_ubi_repo_sets: @@ -826,21 +615,47 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): ), ], } - mock_ubipop_runner.repos.packages = { - "test_rpm": [ - get_test_pkg(name="test_rpm", filename="test_rpm.rpm"), - ], - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="test_debug_pkg", filename="test_debug_pkg.rpm"), - ], - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="test_srpm", filename="test_srpm.src.rpm"), - ], - } + + binary_rpms = [ + UbiUnit( + RpmUnit( + name="test_rpm", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="test_debug_pkg", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="test_srpm", + version="1", + release="2", + arch="x86_64", + filename="test_srpm.src.rpm", + ), + "foo-source", + ) + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) modular_binary = UbiUnit( RpmUnit(name="modular_binary", version="1.0", release="1", arch="x86_64"), @@ -958,21 +773,47 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft ), ], } - mock_ubipop_runner.repos.packages = { - "test_rpm": [ - get_test_pkg(name="rpm_current", filename="rpm_current.rpm"), - ], - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="debug_rpm_current", filename="debug_rpm_current.rpm"), - ], - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="srpm_current", filename="srpm_current.src.rpm"), - ], - } + + binary_rpms = [ + UbiUnit( + RpmUnit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="srpm_current", + version="1", + release="2", + arch="x86_64", + filename="srpm_current.src.rpm", + ), + "foo-source", + ) + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) # pylint: disable=protected-access ( @@ -1072,29 +913,97 @@ def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft): ) ] } - mock_ubipop_runner.repos.packages = { - "test_rpm": [get_test_pkg(name="rpm_current", filename="rpm_current.rpm")] - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="debug_rpm_current", filename="debug_rpm_current.rpm") - ] - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.0-1.src.rpm") - ], - "test_srpm2": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.0-2.src.rpm") - ], - "test_srpm3": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.1-1.src.rpm") - ], - "test_pkg": [get_test_pkg(name="test_pkg", filename="srpm_new.src.rpm")], - "foo_pkg": [get_test_pkg(name="foo_pkg", filename="srpm_new.src.rpm")], - "bar_pkg": [get_test_pkg(name="bar_pkg", filename="srpm_new_next.src.rpm")], - } + binary_rpms = [ + UbiUnit( + RpmUnit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="1", + arch="src", + filename="test_srpm-1.0-1.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="2", + arch="src", + filename="test_srpm-1.0-2.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="2", + arch="src", + filename="test_srpm-1.1-1.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="foo_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="bar_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new_next.src.rpm", + ), + "foo-source", + ), + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) # pylint: disable=protected-access associations, _, _, _ = mock_ubipop_runner._get_pulp_actions( @@ -1162,97 +1071,3 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): # the calls has to be in order calls = [call(["task_id_0"]), call(["task_id_1"])] mock_ubipop_runner.pulp.wait_for_tasks.assert_has_calls(calls) - - -def test_finalize_rpms_output_set(mock_ubipop_runner): - expected_filename = "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" - mock_ubipop_runner.repos.packages["tomcatjss"] = [ - get_test_pkg( - name="tomcatjss", - filename=expected_filename, - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._finalize_rpms_output_set() # pylint: disable=protected-access - - out_packages = mock_ubipop_runner.repos.packages["tomcatjss"] - assert len(out_packages) == 1 - assert out_packages[0].filename == expected_filename - - -def test_finalize_debug_output_set(mock_ubipop_runner): - expected_filename = "tomcatjss-debuginfo-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" - mock_ubipop_runner.repos.debug_rpms["tomcatjss-debuginfo"] = [ - get_test_pkg( - name="tomcatjss-debuginfo", - filename=expected_filename, - ), - get_test_pkg( - name="tomcatjss-debuginfo", - filename="tomcatjss-debuginfo-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._finalize_debug_output_set() # pylint: disable=protected-access - out_packages = mock_ubipop_runner.repos.debug_rpms["tomcatjss-debuginfo"] - assert len(out_packages) == 1 - assert out_packages[0].filename == expected_filename - - -def test_exclude_blacklisted_packages(mock_ubipop_runner): - mock_ubipop_runner.repos.packages["kernel-blacklisted"] = [ - get_test_pkg( - name="kernel-blacklisted", - filename="kernel-blacklisted-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner.repos.pkgs_from_modules = deepcopy( - mock_ubipop_runner.repos.packages - ) - - mock_ubipop_runner.repos.debug_rpms["kernel-blacklisted-debuginfo"] = [ - get_test_pkg( - name="kernel-blacklisted-debuginfo", - filename="kernel-blacklisted-debuginfo-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._exclude_blacklisted_packages() # pylint: disable=protected-access - - assert len(mock_ubipop_runner.repos.packages) == 0 - # no blacklisting from pkgs from mds - assert len(mock_ubipop_runner.repos.pkgs_from_modules) == 1 - assert len(mock_ubipop_runner.repos.debug_rpms) == 0 - - -def test_get_pkgs_from_all_modules(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:7.3.6-1.el8+1944+b6c8e16f.noarch"], - ), - get_test_mod( - name="m2", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:8.4.7-2.el8+1944+b6c8e16f.noarch"], - ), - # intentionally the same pkg as m2 module, pkgs are not to be duplicated - get_test_mod( - name="m3", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:8.4.7-2.el8+1944+b6c8e16f.noarch"], - ), - ] - - pkgs = ( - mock_ubipop_runner._get_pkgs_from_all_modules() - ) # pylint: disable=protected-access - assert len(pkgs) == 2 - assert "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" in pkgs - assert "tomcatjss-8.4.7-2.el8+1944+b6c8e16f.noarch.rpm" in pkgs diff --git a/tests/test_utils.py b/tests/test_utils.py index 0ba5433..03ee339 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from mock import MagicMock -from pubtools.pulplib import YumRepository +from pubtools.pulplib import YumRepository, RpmUnit from ubipop._utils import ( AssociateAction, AssociateActionModuleDefaults, @@ -11,7 +11,9 @@ UnassociateActionModuleDefaults, UnassociateActionModules, UnassociateActionRpms, + vercmp_sort, ) +from ubipop._matcher import UbiUnit def test_raise_not_implemented_pulp_action(): @@ -81,3 +83,40 @@ def test_get_action_unassociate(klass, method): assert "mock." + method in str(associate_action) assert current_units == units assert dst_repo_current.id == dst_repo.id + + +def test_vercmp_sort(): + vercmp_klass = vercmp_sort() + + unit_1 = vercmp_klass( + UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + epoch="1", + arch="x86_64", + ), + None, + ) + ) + + unit_2 = vercmp_klass( + UbiUnit( + RpmUnit( + name="test", + version="10", + release="200", + epoch="1", + arch="x86_64", + ), + None, + ) + ) + + assert (unit_1 < unit_2) is True + assert (unit_1 <= unit_2) is True + assert (unit_1 == unit_2) is False + assert (unit_1 >= unit_2) is False + assert (unit_1 > unit_2) is False + assert (unit_1 != unit_2) is True diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 87f28e3..2347277 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -1,3 +1,4 @@ +from datetime import date import logging import re @@ -20,7 +21,7 @@ UnassociateActionModuleDefaults, UnassociateActionRpms, ) -from ._matcher import ModularMatcher +from ._matcher import ModularMatcher, RpmMatcher _LOG = logging.getLogger("ubipop") @@ -378,151 +379,6 @@ def _match_module_defaults(self): if module_defaults: self.repos.module_defaults[fts[ft]].extend(module_defaults) - def _get_pkgs_from_all_modules(self): - modules = [] - for in_repo_rpm in self.repos.in_repos.rpm: - modules.extend(self.pulp.search_modules(in_repo_rpm)) - pkgs = set() - regex = r"\d+:" - reg = re.compile(regex) - for module in modules: - for pkg in module.packages: - rpm_without_epoch = reg.sub("", pkg) - rpm_filename = rpm_without_epoch + ".rpm" - pkgs.add(rpm_filename) - - return pkgs - - def _match_packages(self, input_repos, packages_dict): - """ - Add matching packages from whitelist - Globbing package name is not supported - """ - modular_pkgs = self._get_pkgs_from_all_modules() - fts = {} - for package_pattern in self.ubiconfig.packages.whitelist: - name = package_pattern.name - arch = None if package_pattern.arch in ("*", None) else package_pattern.arch - for repo in input_repos: - fts[ - (self._executor.submit(self.pulp.search_rpms, repo, name, arch)) - ] = name - - for ft in as_completed(fts): - packages = ft.result() - if packages: - for pkg in packages: - # skip modular packages, those are handled separately - if pkg.filename in modular_pkgs: - continue - - packages_dict[fts[ft]].append(pkg) - - def _match_binary_rpms(self): - self._match_packages(self.repos.in_repos.rpm, self.repos.packages) - - def _match_debug_rpms(self): - self._match_packages(self.repos.in_repos.debug, self.repos.debug_rpms) - - def _parse_blacklist_config(self): - packages_to_exclude = [] - for package_pattern in self.ubiconfig.packages.blacklist: - name_to_parse = package_pattern.name - globbing = "*" in name_to_parse - if globbing: - name = package_pattern.name[:-1] - else: - name = package_pattern.name - arch = None if package_pattern.arch in ("*", None) else package_pattern.arch - - packages_to_exclude.append((name, globbing, arch)) - - return packages_to_exclude - - def _exclude_blacklisted_packages(self): - blacklisted_binary = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.packages.values())) - ) - blacklisted_debug = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.debug_rpms.values())) - ) - - for pkg in blacklisted_binary: - # blacklist only non-modular pkgs - self.repos.packages[pkg.name][:] = [ - _pkg - for _pkg in self.repos.packages.get(pkg.name, []) - if _pkg.is_modular - ] - - # if there is nothing left, remove whole entry for package - if not self.repos.packages[pkg.name]: - self.repos.packages.pop(pkg.name, None) - - for pkg in blacklisted_debug: - # blacklist only non-modular debug pkgs - self.repos.debug_rpms[pkg.name][:] = [ - _pkg - for _pkg in self.repos.debug_rpms.get(pkg.name, []) - if _pkg.is_modular - ] - - # if there is nothing left, remove whole entry for debug package - if not self.repos.debug_rpms[pkg.name]: - self.repos.debug_rpms.pop(pkg.name, None) - - def _finalize_rpms_output_set(self): - for _, packages in self.repos.packages.items(): - self._finalize_output_units(packages, "rpm") - - def _finalize_debug_output_set(self): - for _, packages in self.repos.debug_rpms.items(): - self._finalize_output_units(packages, "rpm") - - def _finalize_output_units(self, units, type_id): - if type_id == "rpm": - self.sort_packages(units) - self.keep_n_latest_packages( - units - ) # with respect to packages referenced by modules - - def _create_srpms_output_set(self): - rpms = chain.from_iterable(self.repos.packages.values()) - binary_source_repo_map = {} - for package in rpms: - if package.sourcerpm is None: - _LOG.warning( - "Package %s doesn't reference its source rpm", package.name - ) - continue - in_repo = [ - r - for r in self.repos.in_repos.rpm - if r.id == package.associate_source_repo_id - ][0] - - if in_repo.id not in binary_source_repo_map: - binary_source_repo_map[in_repo.id] = in_repo.get_source_repository() - - associate_src_repo = binary_source_repo_map[in_repo.id] - - self.repos.source_rpms[package.name].append( - Package( - package.name, - package.sourcerpm, - is_modular=package.is_modular, - src_repo_id=associate_src_repo.id, - ) - ) - - blacklisted_srpms = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.source_rpms.values())) - ) - - for pkg in blacklisted_srpms: - if not pkg.is_modular: - self.repos.source_rpms.pop(pkg.name, None) - def _determine_pulp_actions(self, units, current, diff_f, extra_units=None): expected = list(units) if extra_units: @@ -546,9 +402,8 @@ def _get_pulp_actions_md_defaults(self, module_defaults, current): ) def _get_pulp_actions_pkgs(self, pkgs, current, modular_pkgs): - pkgs_list = list(chain.from_iterable(pkgs.values())) return self._determine_pulp_actions( - pkgs_list, current, self._diff_packages_by_filename, modular_pkgs + pkgs, current, self._diff_packages_by_filename, modular_pkgs ) def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): @@ -558,7 +413,7 @@ def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): """ uniq_srpms = {} - all_pkgs = list(chain.from_iterable(pkgs.values())) + list(modular) + all_pkgs = list(pkgs) + list(modular) # filter out packages that share same source rpm for pkg in all_pkgs: @@ -589,6 +444,7 @@ def _get_pulp_actions( Content that needs unassociation: unit is in current but not in expected No action: unit is in current and in expected """ + modules_assoc, modules_unassoc = self._get_pulp_actions_mds( self.repos.modules, current_modules_ft.result() ) @@ -668,19 +524,14 @@ def run_ubi_population(self): current_debug_rpms_ft, ) = self._get_current_content() - # start async querying for modulemds and modular packages + # start async querying for modulemds and modular and non-modular packages mm = ModularMatcher(self.repos.in_repos, self.ubiconfig.modules).run() - self.repos.modules = mm.modules - - self._match_binary_rpms() - if self.repos.out_repos.debug: - self._match_debug_rpms() - self._exclude_blacklisted_packages() + rm = RpmMatcher(self.repos.in_repos, self.ubiconfig).run() - # only non-modular packages - self._finalize_rpms_output_set() - self._finalize_debug_output_set() - self._create_srpms_output_set() + self.repos.modules = mm.modules + self.repos.packages = rm.binary_rpms + self.repos.debug_rpms = rm.debug_rpms + self.repos.source_rpms = rm.source_rpms self._match_module_defaults() @@ -856,60 +707,3 @@ def _publish_out_repos(self): if repo.result(): fts.append(repo.publish(options)) return fts - - def get_blacklisted_packages(self, package_list): - """ - Finds blacklisted packages in output sets - """ - blacklisted_pkgs = [] - for pattern_name, globbing, pattern_arch in self._parse_blacklist_config(): - for package in package_list: - name, _, _, _, arch = split_filename(package.filename) - blacklisted = False - if globbing: - if name.startswith(pattern_name): - blacklisted = True - else: - if name == pattern_name: - blacklisted = True - - if pattern_arch: - if arch != pattern_arch: - blacklisted = False - - if blacklisted: - blacklisted_pkgs.append(package) - - return blacklisted_pkgs - - def sort_packages(self, packages): - """ - Sort packages by vercmp - """ - packages.sort() - - def keep_n_latest_packages(self, packages, n=1): - """ - Keep n latest non-modular packages. - - Arguments: - packages (List[Package]): Sorted, oldest goes first - - Keyword arguments: - n (int): Number of non-modular package versions to keep - - Returns: - None. The packages list is changed in-place - """ - # Use a queue of n elements per arch - pkgs_per_arch = defaultdict(lambda: deque(maxlen=n)) - - for package in packages: - _, _, _, _, arch = split_filename(package.filename) - pkgs_per_arch[arch].append(package) - - latest_pkgs_per_arch = [ - pkg for pkg in chain.from_iterable(pkgs_per_arch.values()) - ] - - packages[:] = latest_pkgs_per_arch diff --git a/ubipop/_matcher.py b/ubipop/_matcher.py index 158612b..e83a8c5 100644 --- a/ubipop/_matcher.py +++ b/ubipop/_matcher.py @@ -1,12 +1,18 @@ import os +from itertools import chain +from collections import defaultdict, deque +from concurrent.futures import as_completed from pubtools.pulplib import Criteria +from pubtools.pulplib import Matcher as PulpLibMatcher from more_executors.futures import f_flat_map, f_return, f_sequence, f_proxy from more_executors import Executors -from ubipop._utils import split_filename - +from ubipop._utils import split_filename, vercmp_sort BATCH_SIZE = int(os.getenv("UBIPOP_BATCH_SIZE", "250")) +# need to set significantly lower batches for general rpm search +# otherwise db may very likely hit hit OOM error. +BATCH_SIZE_RPM = int(os.getenv("UBIPOP_BATCH_SIZE_RPM", "15")) class UbiUnit(object): @@ -45,6 +51,10 @@ def __init__(self, input_repos, ubi_config, workers=8): # we use executor from pulplib self._executor = Executors.thread_pool(max_workers=workers) + self.binary_rpms = None + self.debug_rpms = None + self.source_rpms = None + def run(self): """ This method needs to be implemented in subclasses and should @@ -53,11 +63,14 @@ def run(self): """ raise NotImplementedError - def _search_units(self, repo, criteria_list, content_type_id): + def _search_units( + self, repo, criteria_list, content_type_id, batch_size_override=None + ): """ Search for units of one content type associated with given repository by criteria. """ units = set() + batch_size = batch_size_override or BATCH_SIZE def handle_results(page): for unit in page.data: @@ -69,8 +82,8 @@ def handle_results(page): criteria_split = [] - for start in range(0, len(criteria_list), BATCH_SIZE): - criteria_split.append(criteria_list[start : start + BATCH_SIZE]) + for start in range(0, len(criteria_list), batch_size): + criteria_split.append(criteria_list[start : start + batch_size]) fts = [] for criteria_batch in criteria_split: @@ -99,36 +112,67 @@ def _create_or_criteria(self, fields, values): if len(val_tuple) != len(fields): raise ValueError for index, field in enumerate(fields): + inner_and_criteria.append(Criteria.with_field(field, val_tuple[index])) or_criteria.append(Criteria.and_(*inner_and_criteria)) return or_criteria - def _search_units_per_repos(self, or_criteria, repos, content_type): + def _search_units_per_repos( + self, or_criteria, repos, content_type, batch_size_override=None + ): units = [] for repo in repos: - units.append(self._search_units(repo, or_criteria, content_type)) + units.append( + self._search_units( + repo, + or_criteria, + content_type, + batch_size_override=batch_size_override, + ) + ) return f_proxy(f_flat_map(f_sequence(units), flatten_list_of_sets)) - def _search_rpms(self, or_criteria, repos): - return self._search_units_per_repos(or_criteria, repos, content_type="rpm") + def _search_rpms(self, or_criteria, repos, batch_size_override=None): + return self._search_units_per_repos( + or_criteria, + repos, + content_type="rpm", + batch_size_override=batch_size_override, + ) - def _search_srpms(self, or_criteria, repos): - return self._search_units_per_repos(or_criteria, repos, content_type="srpm") + def _search_srpms(self, or_criteria, repos, batch_size_override=None): + return self._search_units_per_repos( + or_criteria, + repos, + content_type="srpm", + batch_size_override=batch_size_override, + ) def _search_moludemds(self, or_criteria, repos): return self._search_units_per_repos(or_criteria, repos, content_type="modulemd") + def _get_srpms_criteria(self): + filenames = [] + for rpms_list in as_completed([self.binary_rpms, self.debug_rpms]): + for pkg in rpms_list: + if pkg.sourcerpm is None: + continue + # _LOG.warning( + # "Package %s doesn't reference its source rpm", package.name + # ) + filenames.append((pkg.sourcerpm,)) + + pkgs_or_criteria = self._create_or_criteria(("filename",), filenames) + return pkgs_or_criteria + class ModularMatcher(Matcher): def __init__(self, input_repos, ubi_config): super(ModularMatcher, self).__init__(input_repos, ubi_config) self.modules = None - self.binary_rpms = None - self.debug_rpms = None - self.source_rpms = None def run(self): """Asynchronously creates criteria for pulp queries and @@ -159,9 +203,7 @@ def run(self): self._search_rpms, rpms_criteria, self._input_repos.debug ) ) - srpms_criteria = f_proxy( - self._executor.submit(self._get_modular_srpms_criteria) - ) + srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) self.source_rpms = f_proxy( self._executor.submit( self._search_srpms, srpms_criteria, self._input_repos.source @@ -175,12 +217,6 @@ def _get_modular_rpms_criteria(self): pkgs_or_criteria = self._create_or_criteria(("filename",), filenames_to_search) return pkgs_or_criteria - def _get_modular_srpms_criteria(self): - non_source_pkg = list(self.binary_rpms) + list(self.debug_rpms) - filenames = [(pkg.sourcerpm,) for pkg in non_source_pkg] - pkgs_or_criteria = self._create_or_criteria(("filename",), filenames) - return pkgs_or_criteria - def _get_modulemds_criteria(self): criteria_values = [] for module in self._ubi_config: @@ -204,7 +240,7 @@ def _get_modulemd_output_set(self, modules): name_stream_modules_map.setdefault(key, []).append(modulemd) out = [] - # sort modulemds and keep N latest versions of them + # sort rpms and keep N latest versions of them for module_list in name_stream_modules_map.values(): module_list.sort(key=lambda module: module.version) self._keep_n_latest_modules(module_list) @@ -257,6 +293,176 @@ def _modular_rpms_filenames(self, modules): return filenames +class RpmMatcher(Matcher): + def __init__(self, input_repos, ubi_config): + super(RpmMatcher, self).__init__(input_repos, ubi_config) + self.binary_rpms = None + self.debug_rpms = None + self.source_rpms = None + + def run(self): + batch_size_override = BATCH_SIZE_RPM + + modular_rpm_filenames = f_proxy(self._get_pkgs_from_all_modules()) + rpms_criteria = f_proxy(self._executor.submit(self._get_rpms_criteria)) + + binary_rpms = f_proxy( + self._executor.submit( + self._search_rpms, + rpms_criteria, + self._input_repos.rpm, + batch_size_override, + ) + ) + + debug_rpms = f_proxy( + self._executor.submit( + self._search_rpms, + rpms_criteria, + self._input_repos.debug, + batch_size_override, + ) + ) + + self.binary_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, binary_rpms, modular_rpm_filenames + ) + ) + self.debug_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, debug_rpms, modular_rpm_filenames + ) + ) + + srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) + source_rpms = f_proxy( + self._executor.submit( + self._search_srpms, + srpms_criteria, + self._input_repos.source, + batch_size_override, + ) + ) + + self.source_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, source_rpms, modular_rpm_filenames + ) + ) + + return self + + def _get_rpms_criteria(self): + criteria_values = [] + + for package_pattern in self._ubi_config.packages.whitelist: + # skip src packages, they are searched seprately + if package_pattern.arch == "src": + continue + arch = ( + PulpLibMatcher.exists() + if package_pattern.arch in ("*", None) + else package_pattern.arch + ) + criteria_values.append((package_pattern.name, arch)) + + fields = ("name", "arch") + or_criteria = self._create_or_criteria(fields, criteria_values) + return or_criteria + + def _get_pkgs_from_all_modules(self): + # search for modulesmds in all input repos + # extract filenames + def extract_modular_filenames(): + modular_rpm_filenames = set() + for module in modules: + modular_rpm_filenames |= set(module.artifacts_filenames) + + return modular_rpm_filenames + + modules = self._search_moludemds([Criteria.true()], self._input_repos.rpm) + return self._executor.submit(extract_modular_filenames) + + def _get_rpm_output_set(self, rpms, modular_rpm_filenames): + blacklist_parsed = self._parse_blacklist_config() + name_rpms_maps = {} + + def is_blacklisted(rpm): + for name, globbing, arch in blacklist_parsed: + blacklisted = False + if globbing: + if rpm.name.startswith(name): + blacklisted = True + else: + if rpm.name == name: + blacklisted = True + if arch: + if rpm.arch != arch: + blacklisted = False + + if blacklisted: + return blacklisted + + for rpm in rpms: + # skip modular rpms + if rpm.filename in modular_rpm_filenames: + continue + + if is_blacklisted(rpm): + continue + + name_rpms_maps.setdefault(rpm.name, []).append(rpm) + + out = [] + # sort rpms and keep N latest versions of them + for rpm_list in name_rpms_maps.values(): + rpm_list.sort(key=vercmp_sort()) + self._keep_n_latest_rpms(rpm_list) + out.extend(rpm_list) + + return out + + def _keep_n_latest_rpms(self, rpms, n=1): + """ + Keep n latest non-modular rpms. + + Arguments: + rpms (List[Rpm]): Sorted, oldest goes first + + Keyword arguments: + n (int): Number of non-modular package versions to keep + + Returns: + None. The packages list is changed in-place + """ + # Use a queue of n elements per arch + pkgs_per_arch = defaultdict(lambda: deque(maxlen=n)) + + for rpm in rpms: + pkgs_per_arch[rpm.arch].append(rpm) + + latest_pkgs_per_arch = [ + pkg for pkg in chain.from_iterable(pkgs_per_arch.values()) + ] + + rpms[:] = latest_pkgs_per_arch + + def _parse_blacklist_config(self): + packages_to_exclude = [] + for package_pattern in self._ubi_config.packages.blacklist: + globbing = package_pattern.name.endswith("*") + if globbing: + name = package_pattern.name[:-1] + else: + name = package_pattern.name + arch = None if package_pattern.arch in ("*", None) else package_pattern.arch + + packages_to_exclude.append((name, globbing, arch)) + + return packages_to_exclude + + def flatten_list_of_sets(list_of_sets): out = set() for one_set in list_of_sets: diff --git a/ubipop/_utils.py b/ubipop/_utils.py index 2a3d025..79318a8 100644 --- a/ubipop/_utils.py +++ b/ubipop/_utils.py @@ -1,3 +1,6 @@ +from rpm import labelCompare as label_compare +from rpm import labelCompare as label_compare + # borrowed from https://github.com/rpm-software-management/yum def split_filename(filename): """ @@ -137,3 +140,29 @@ class UnassociateActionRpms(PulpAction): def get_actions(self, pulp_client_inst): return [(pulp_client_inst.unassociate_packages, self.dst_repo, self.units)] + + +def vercmp_sort(): + class K(object): + def __init__(self, package): + self.evr_tuple = (package.epoch, package.version, package.release) + + def __lt__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) < 0 + + def __gt__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) > 0 + + def __eq__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) == 0 + + def __le__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) <= 0 + + def __ge__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) >= 0 + + def __ne__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) != 0 + + return K