diff --git a/.pylintrc b/.pylintrc index 916bcc9..3d62b6b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -99,4 +99,6 @@ disable=print-statement, # skip linting Python 3 features super-with-arguments, raise-missing-from, + # skip check for non-snake_case names + invalid-name, diff --git a/tests/test_matcher.py b/tests/test_matcher.py index 761b45b..ff9e34e 100644 --- a/tests/test_matcher.py +++ b/tests/test_matcher.py @@ -1,6 +1,6 @@ +from operator import attrgetter import pytest -from operator import attrgetter from more_executors.futures import f_proxy, f_return from pubtools.pulplib import ( RpmUnit, @@ -11,8 +11,15 @@ ) from ubiconfig import UbiConfig -from ubipop._matcher import UbiUnit, Matcher, flatten_list_of_sets, ModularMatcher +from ubipop._matcher import ( + UbiUnit, + Matcher, + flatten_list_of_sets, + ModularMatcher, + RpmMatcher, +) from ubipop import RepoSet +from ubipop._utils import vercmp_sort @pytest.fixture(name="pulp") @@ -23,6 +30,7 @@ def fake_pulp(): @pytest.fixture(name="ubi_config") def fake_ubi_config(): config_dict = { + "arches": ["src"], "modules": { "include": [ { @@ -32,7 +40,14 @@ def fake_ubi_config(): } ] }, - "packages": {}, + "packages": { + "include": ["test.*", "something_else.src"], + "exclude": [ + "excluded_with_globbing*", + "excluded_package.*", + "excluded_with_arch.src", + ], + }, "content_sets": {}, } yield UbiConfig.load_from_dict(config_dict, "fake/config.yaml") @@ -504,7 +519,7 @@ def test_get_modular_srpms_criteria(ubi_config): matcher.binary_rpms = f_proxy(f_return(set([unit_1]))) matcher.debug_rpms = f_proxy(f_return(set([unit_2]))) - criteria = matcher._get_modular_srpms_criteria() + criteria = matcher._get_srpms_criteria() # there should be 1 criteria created assert len(criteria) == 2 # it should be instance of Criteria @@ -634,6 +649,347 @@ def test_modular_matcher_run(pulp, ubi_config): assert rpm.associate_source_repo_id == "source_repo" +def test_parse_blacklist_config(ubi_config): + """Tests parsing blacklist for ubi-config""" + matcher = RpmMatcher(None, ubi_config) + blacklist_parsed = sorted(matcher._parse_blacklist_config()) + + assert len(blacklist_parsed) == 3 + + expected_blacklist_parsed = sorted( + [ + ("excluded_package", False, None), + ("excluded_with_globbing", True, None), + ("excluded_with_arch", False, "src"), + ] + ) + + assert blacklist_parsed == expected_blacklist_parsed + + +def test_keep_n_latest_rpms(): + + """Test keeping only the latest version of modulemd""" + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + + unit_2 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + + matcher = RpmMatcher(None, None) + rpms = [unit_1, unit_2] + rpms.sort(key=vercmp_sort()) + matcher._keep_n_latest_rpms(rpms) + + # there should only one modulemd + assert len(rpms) == 1 + # with the highest number of version + assert rpms[0].version == "11" + + +def test_keep_n_latest_rpms_multiple_arches(): + + """Test keeping only the latest version of modulemd""" + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + + unit_2 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + unit_3 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="i686", + ), + None, + ) + + matcher = RpmMatcher(None, None) + rpms = [unit_1, unit_2, unit_3] + rpms.sort(key=vercmp_sort()) + matcher._keep_n_latest_rpms(rpms) + + # there should only one modulemd + assert len(rpms) == 2 + # with the highest number of version + assert rpms[0].version == "11" + assert rpms[0].arch == "x86_64" + + # with the highest number of version + assert rpms[1].version == "10" + assert rpms[1].arch == "i686" + + +def test_get_rpm_output_set(ubi_config): + """tests getting rpm output set from RpmMatcher""" + matcher = RpmMatcher(None, ubi_config) + # this unit won't be in output set, there is a newver one - unit_4 with higher version + unit_1 = UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + # this one will be excluded using blacklist + unit_2 = UbiUnit( + RpmUnit( + name="excluded_with_globbing123456789", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + # this one will be excluded using blacklist + unit_3 = UbiUnit( + RpmUnit( + name="excluded_package", + version="10", + release="20", + arch="x86_64", + ), + None, + ) + # this it the only one that will in output set + unit_4 = UbiUnit( + RpmUnit( + name="test", + version="11", + release="20", + arch="x86_64", + ), + None, + ) + # this one is excluded because it's a modular package + unit_5 = UbiUnit( + RpmUnit( + name="modular_package", + version="10", + release="20", + arch="x86_64", + filename="modular_package.rpm", + ), + None, + ) + + modular_pkgs_filenames = f_proxy(f_return(set(["modular_package.rpm"]))) + rpms = [unit_1, unit_2, unit_3, unit_4, unit_5] + output = matcher._get_rpm_output_set(rpms, modular_pkgs_filenames) + + # in the output set there is only one unit, according to the rules + assert len(output) == 1 + assert output[0].name == "test" + assert output[0].version == "11" + + +def test_get_pkgs_from_all_modules(pulp): + """tests getting pkgs filenames from all available modulemd units""" + repo = YumRepository( + id="test_repo_1", + ) + repo.__dict__["_client"] = pulp.client + unit_1 = ModulemdUnit( + name="test", + stream="10", + version=100, + context="abcdef", + arch="x86_64", + artifacts=[ + "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.src", + "perl-version-7:0.99.24-441.module+el8.3.0+6718+7f269185.x86_64", + ], + ) + unit_2 = ModulemdUnit( + name="test", + stream="20", + version=100, + context="abcdef", + arch="x86_64", + artifacts=[ + "perl-version-7:1.99.24-441.module+el8.4.0+9911+7f269185.src", + "perl-version-7:1.99.24-441.module+el8.4.0+9911+7f269185.x86_64", + ], + ) + + pulp.insert_repository(repo) + pulp.insert_units(repo, [unit_1, unit_2]) + repos_set = RepoSet(rpm=[repo], debug=[], source=[]) + matcher = RpmMatcher(repos_set, None) + modular_filenames = matcher._get_pkgs_from_all_modules().result() + + # there are 4 filenames according from 2 modulemd units + expected_filenames = set( + [ + "perl-version-0.99.24-441.module+el8.3.0+6718+7f269185.src.rpm", + "perl-version-0.99.24-441.module+el8.3.0+6718+7f269185.x86_64.rpm", + "perl-version-1.99.24-441.module+el8.4.0+9911+7f269185.src.rpm", + "perl-version-1.99.24-441.module+el8.4.0+9911+7f269185.x86_64.rpm", + ] + ) + + assert len(modular_filenames) == 4 + assert modular_filenames == expected_filenames + + +def test_get_rpms_criteria(ubi_config): + """tests proper creation of criteria for rpms search in RpmMatcher""" + + matcher = RpmMatcher(None, ubi_config) + criteria = matcher._get_rpms_criteria() + # there should be 1 criterium created based on ubi config + # we skip package with src "arch", those are queried separately + assert len(criteria) == 1 + # it should be instance of Criteria + for crit in criteria: + assert isinstance(crit, Criteria) + # let's not test internal structure of criteria, that's responsibility of pulplib + + +def test_rpm_matcher_run(pulp, ubi_config): + """Test run() method which asynchronously creates criteria for queries to pulp + and exectues those query. Finally it sets up public attrs of the RpmMatcher + object that can be used in ubipop""" + + repo_1 = YumRepository( + id="binary_repo", + ) + repo_1.__dict__["_client"] = pulp.client + + repo_2 = YumRepository( + id="debug_repo", + ) + repo_2.__dict__["_client"] = pulp.client + repo_3 = YumRepository( + id="source_repo", + ) + repo_3.__dict__["_client"] = pulp.client + + # binary - will be in output set + unit_1 = RpmUnit( + name="test", + version="1.0", + release="1", + arch="x86_64", + filename="test-1.0-1.x86_64.rpm", + sourcerpm="test-1.0-1.src.rpm", + ) + # debug - will be in output set + unit_2 = RpmUnit( + name="test", + version="1.0", + release="1", + arch="x86_64", + filename="test-debug-1.0-1.x86_64.rpm", + ) + # source - will be in output set + unit_3 = RpmUnit( + name="test-src", + version="1.0", + release="1", + arch="src", + filename="test-1.0-1.src.rpm", + content_type_id="srpm", + ) + + # modular - will be skipped + unit_4 = RpmUnit( + name="test", + version="100.0", + release="1", + arch="x86_64", + filename="test-100.0-1.x86_64.rpm", + sourcerpm="test-100.0-1.src.rpm", + ) + # blacklisted - will be also skipped + unit_5 = RpmUnit( + name="excluded_package", + version="1.0", + release="1", + arch="x86_64", + filename="excluded_package-1.0-1.x86_64.rpm", + sourcerpm="excluded_package-1.0-1.src.rpm", + ) + # non-matching - not in output + unit_6 = RpmUnit( + name="no_match", + version="1.0", + release="1", + arch="x86_64", + filename="no_match-1.0-1.x86_64.rpm", + sourcerpm="no_match-1.0-1.src.rpm", + ) + + modulemd = ModulemdUnit( + name="fake_name", + stream="fake_stream", + version=100, + context="abcd", + arch="x86_64", + artifacts=[ + "test-0:100.0-1.x86_64", + ], + ) + pulp.insert_repository(repo_1) + pulp.insert_repository(repo_2) + pulp.insert_repository(repo_3) + pulp.insert_units(repo_1, [unit_1, modulemd, unit_4, unit_5, unit_6]) + pulp.insert_units(repo_2, [unit_2]) + pulp.insert_units(repo_3, [unit_3]) + + repos_set = RepoSet(rpm=[repo_1], debug=[repo_2], source=[repo_3]) + matcher = RpmMatcher(repos_set, ubi_config) + matcher.run() + # each public attribute is properly set with one unit + assert len(matcher.binary_rpms) == 1 + assert len(matcher.debug_rpms) == 1 + assert len(matcher.source_rpms) == 1 + + # each unit is properly queried + rpm = matcher.binary_rpms.pop() + assert rpm.filename == "test-1.0-1.x86_64.rpm" + assert rpm.associate_source_repo_id == "binary_repo" + + rpm = matcher.debug_rpms.pop() + assert rpm.filename == "test-debug-1.0-1.x86_64.rpm" + assert rpm.associate_source_repo_id == "debug_repo" + + rpm = matcher.source_rpms.pop() + assert rpm.filename == "test-1.0-1.src.rpm" + assert rpm.associate_source_repo_id == "source_repo" + + def test_flatten_list_of_sets(): """Test helper function that flattens list of sets into one set""" set_1 = set([1, 2, 3]) diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 4d45d97..34247ac 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -1,3 +1,6 @@ +from datetime import datetime +from operator import attrgetter + import logging import os import shutil @@ -6,9 +9,6 @@ import pytest import ubiconfig -from copy import deepcopy -from datetime import datetime -from operator import attrgetter from pubtools.pulplib import ( YumRepository, FakeController, @@ -354,45 +354,6 @@ def test_get_ubi_repo_sets(get_debug_repository, get_source_repository): assert output_repos.debug.id == "ubi_debug" -def test_get_blacklisted_packages_match_name_glob(mock_ubipop_runner): - pkg_name = "foo-pkg" - test_pkg_list = [ - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.noarch.rpm".format(name=pkg_name), - ), - get_test_pkg( - name="no-match-foo-pkg", - filename="no-match-foo-pkg-3.0.6-4.el7.noarch.rpm", - ), - ] - - blacklist = mock_ubipop_runner.get_blacklisted_packages(test_pkg_list) - - assert len(blacklist) == 1 - assert blacklist[0].name == pkg_name - - -def test_get_blacklisted_packages_match_arch(mock_ubipop_runner): - pkg_name = "foo-arch-test" - test_pkg_list = [ - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.noarch.rpm".format(name=pkg_name), - ), - get_test_pkg( - name=pkg_name, - filename="{name}-3.0.6-4.el7.x86_64.rpm".format(name=pkg_name), - ), - ] - - blacklist = mock_ubipop_runner.get_blacklisted_packages(test_pkg_list) - - assert len(blacklist) == 1 - assert blacklist[0].name == pkg_name - assert "x86_64" in blacklist[0].filename - - def _get_search_rpms_side_effect(package_name_or_filename_or_list, debug_only=False): def _f(*args, **kwargs): if debug_only and "debug" not in args[0].id: @@ -424,62 +385,6 @@ def _f(*args, **kwargs): return _f -def test_match_binary_rpms_skip_modular(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=["foo-2-pkg-0:7.3.6-1.el8+1944+b6c8e16f.noarch", "foo-pkg0:"], - ), - ] - - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - "foo-pkg", - ) - - mock_ubipop_runner._match_binary_rpms() # pylint: disable=protected-access - # there are no packages stored, modular ones were skipped even if they match - # by ubi_config - assert len(mock_ubipop_runner.repos.packages) == 0 - - -def test_match_binary_rpms(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=[ - "foo-2-pkg-0:7.3.6-1.el8+1944+b6c8e16f.noarch", - ], - ), - ] - - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - "foo-pkg", - ) - - mock_ubipop_runner._match_binary_rpms() # pylint: disable=protected-access - - current_pkg = mock_ubipop_runner.repos.packages["foo-pkg"][0] - assert len(mock_ubipop_runner.repos.packages) == 1 - assert current_pkg.name == "foo-pkg" - assert current_pkg.filename == "foo-pkg.rpm" - # we skip handling modular packages in _match_binary_rpms() method completely - # by default is_modular attr is set to False - assert current_pkg.is_modular is False - - -def test_match_debug_rpms(mock_ubipop_runner): - package_name = "foo-pkg-debuginfo" - mock_ubipop_runner.pulp.search_rpms.side_effect = _get_search_rpms_side_effect( - package_name - ) - mock_ubipop_runner._match_debug_rpms() # pylint: disable=protected-access - - assert len(mock_ubipop_runner.repos.debug_rpms) == 1 - assert mock_ubipop_runner.repos.debug_rpms[package_name][0].name == package_name - - def test_match_module_defaults(mock_ubipop_runner): unit = UbiUnit( ModulemdUnit( @@ -526,62 +431,6 @@ def test_diff_modules(mock_ubipop_runner): assert diff[0].name == "1" -def test_keep_n_newest_packages(mock_ubipop_runner): - packages = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.8-1.el8+1944+b6c8e16f.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - packages.sort() - mock_ubipop_runner.keep_n_latest_packages(packages) - - assert len(packages) == 1 - assert "7.3.8" in packages[0].filename - - -def test_keep_n_newest_packages_multi_arch(mock_ubipop_runner): - packages = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.noarch.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.x86_64.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.i686.rpm", - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.5-1.i686.rpm", - ), - ] - - packages.sort() - mock_ubipop_runner.keep_n_latest_packages(packages) - assert len(packages) == 3 - - arches_expected = ["noarch", "x86_64", "i686"] - arches_current = [] - - for pkg in packages: - _, _, _, _, arch = split_filename(pkg.filename) - arches_current.append(arch) - - assert sorted(arches_current) == sorted(arches_expected) - - @pytest.mark.parametrize( "rhel_repo_set, ubi_repo_set, fail", [ @@ -686,66 +535,6 @@ def test_ubipopulate_load_all_ubiconfig(mocked_ubiconfig_load_all): assert ubipop.ubiconfig_list[0].file_name == "test" -def test_create_srpms_output_set(mock_ubipop_runner): - mock_ubipop_runner.repos.packages["foo"] = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.src.rpm", - src_repo_id="foo-rpms", - ), - # blacklisted - get_test_pkg( - name="kernel", - filename="kernel-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="kernel.src.rpm", - src_repo_id="foo-rpms", - ), - # blacklisted but referenced in some module - get_test_pkg( - name="foo-pkg", - filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - sourcerpm_filename="foo-pkg-7.3.6-1.el8+1944+b6c8e16f.src.rpm", - is_modular=True, - src_repo_id="foo-rpms", - ), - ] - - mock_ubipop_runner._create_srpms_output_set() # pylint: disable=protected-access - - out_srpms = mock_ubipop_runner.repos.source_rpms - assert len(out_srpms) == 2 - assert "tomcatjss" and "foo-pkg" in out_srpms - assert ( - out_srpms["tomcatjss"][0].filename - == "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.src.rpm" - ) - assert ( - out_srpms["foo-pkg"][0].filename == "foo-pkg-7.3.6-1.el8+1944+b6c8e16f.src.rpm" - ) - - -def test_create_srpms_output_set_missings_srpm_reference( - capsys, set_logging, mock_ubipop_runner -): - set_logging.addHandler(logging.StreamHandler(sys.stdout)) - mock_ubipop_runner.repos.packages["foo"] = [ - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._create_srpms_output_set() # pylint: disable=protected-access - - out_srpms = mock_ubipop_runner.repos.source_rpms - assert len(out_srpms) == 0 - out, err = capsys.readouterr() - - assert err == "" - assert out.strip() == "Package tomcatjss doesn't reference its source rpm" - - @pytest.fixture(name="mock_get_repo_pairs") def fixture_mock_get_repo_pairs(ubi_repo_set): with patch("ubipop.UbiPopulate._get_ubi_repo_sets") as get_ubi_repo_sets: @@ -826,21 +615,47 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): ), ], } - mock_ubipop_runner.repos.packages = { - "test_rpm": [ - get_test_pkg(name="test_rpm", filename="test_rpm.rpm"), - ], - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="test_debug_pkg", filename="test_debug_pkg.rpm"), - ], - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="test_srpm", filename="test_srpm.src.rpm"), - ], - } + + binary_rpms = [ + UbiUnit( + RpmUnit( + name="test_rpm", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="test_debug_pkg", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="test_srpm", + version="1", + release="2", + arch="x86_64", + filename="test_srpm.src.rpm", + ), + "foo-source", + ) + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) modular_binary = UbiUnit( RpmUnit(name="modular_binary", version="1.0", release="1", arch="x86_64"), @@ -958,21 +773,47 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft ), ], } - mock_ubipop_runner.repos.packages = { - "test_rpm": [ - get_test_pkg(name="rpm_current", filename="rpm_current.rpm"), - ], - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="debug_rpm_current", filename="debug_rpm_current.rpm"), - ], - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="srpm_current", filename="srpm_current.src.rpm"), - ], - } + + binary_rpms = [ + UbiUnit( + RpmUnit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="srpm_current", + version="1", + release="2", + arch="x86_64", + filename="srpm_current.src.rpm", + ), + "foo-source", + ) + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) # pylint: disable=protected-access ( @@ -1072,29 +913,97 @@ def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft): ) ] } - mock_ubipop_runner.repos.packages = { - "test_rpm": [get_test_pkg(name="rpm_current", filename="rpm_current.rpm")] - } - mock_ubipop_runner.repos.debug_rpms = { - "test_debug_pkg": [ - get_test_pkg(name="debug_rpm_current", filename="debug_rpm_current.rpm") - ] - } - mock_ubipop_runner.repos.source_rpms = { - "test_srpm": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.0-1.src.rpm") - ], - "test_srpm2": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.0-2.src.rpm") - ], - "test_srpm3": [ - get_test_pkg(name="test_srpm", filename="test_srpm-1.1-1.src.rpm") - ], - "test_pkg": [get_test_pkg(name="test_pkg", filename="srpm_new.src.rpm")], - "foo_pkg": [get_test_pkg(name="foo_pkg", filename="srpm_new.src.rpm")], - "bar_pkg": [get_test_pkg(name="bar_pkg", filename="srpm_new_next.src.rpm")], - } + binary_rpms = [ + UbiUnit( + RpmUnit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + ), + "foo-rpms", + ) + ] + debug_rpms = [ + UbiUnit( + RpmUnit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + ), + "foo-debug", + ) + ] + source_rpms = [ + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="1", + arch="src", + filename="test_srpm-1.0-1.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="2", + arch="src", + filename="test_srpm-1.0-2.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_srpm", + version="1.0", + release="2", + arch="src", + filename="test_srpm-1.1-1.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="test_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="foo_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new.src.rpm", + ), + "foo-source", + ), + UbiUnit( + RpmUnit( + name="bar_pkg", + version="1", + release="2", + arch="src", + filename="srpm_new_next.src.rpm", + ), + "foo-source", + ), + ] + + mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) # pylint: disable=protected-access associations, _, _, _ = mock_ubipop_runner._get_pulp_actions( @@ -1162,97 +1071,3 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): # the calls has to be in order calls = [call(["task_id_0"]), call(["task_id_1"])] mock_ubipop_runner.pulp.wait_for_tasks.assert_has_calls(calls) - - -def test_finalize_rpms_output_set(mock_ubipop_runner): - expected_filename = "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" - mock_ubipop_runner.repos.packages["tomcatjss"] = [ - get_test_pkg( - name="tomcatjss", - filename=expected_filename, - ), - get_test_pkg( - name="tomcatjss", - filename="tomcatjss-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._finalize_rpms_output_set() # pylint: disable=protected-access - - out_packages = mock_ubipop_runner.repos.packages["tomcatjss"] - assert len(out_packages) == 1 - assert out_packages[0].filename == expected_filename - - -def test_finalize_debug_output_set(mock_ubipop_runner): - expected_filename = "tomcatjss-debuginfo-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" - mock_ubipop_runner.repos.debug_rpms["tomcatjss-debuginfo"] = [ - get_test_pkg( - name="tomcatjss-debuginfo", - filename=expected_filename, - ), - get_test_pkg( - name="tomcatjss-debuginfo", - filename="tomcatjss-debuginfo-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._finalize_debug_output_set() # pylint: disable=protected-access - out_packages = mock_ubipop_runner.repos.debug_rpms["tomcatjss-debuginfo"] - assert len(out_packages) == 1 - assert out_packages[0].filename == expected_filename - - -def test_exclude_blacklisted_packages(mock_ubipop_runner): - mock_ubipop_runner.repos.packages["kernel-blacklisted"] = [ - get_test_pkg( - name="kernel-blacklisted", - filename="kernel-blacklisted-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner.repos.pkgs_from_modules = deepcopy( - mock_ubipop_runner.repos.packages - ) - - mock_ubipop_runner.repos.debug_rpms["kernel-blacklisted-debuginfo"] = [ - get_test_pkg( - name="kernel-blacklisted-debuginfo", - filename="kernel-blacklisted-debuginfo-7.3.5-1.el8+1944+b6c8e16f.noarch.rpm", - ), - ] - - mock_ubipop_runner._exclude_blacklisted_packages() # pylint: disable=protected-access - - assert len(mock_ubipop_runner.repos.packages) == 0 - # no blacklisting from pkgs from mds - assert len(mock_ubipop_runner.repos.pkgs_from_modules) == 1 - assert len(mock_ubipop_runner.repos.debug_rpms) == 0 - - -def test_get_pkgs_from_all_modules(mock_ubipop_runner): - mock_ubipop_runner.pulp.search_modules.return_value = [ - get_test_mod( - name="m1", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:7.3.6-1.el8+1944+b6c8e16f.noarch"], - ), - get_test_mod( - name="m2", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:8.4.7-2.el8+1944+b6c8e16f.noarch"], - ), - # intentionally the same pkg as m2 module, pkgs are not to be duplicated - get_test_mod( - name="m3", - profiles={"prof1": ["tomcatjss"]}, - packages=["tomcatjss-0:8.4.7-2.el8+1944+b6c8e16f.noarch"], - ), - ] - - pkgs = ( - mock_ubipop_runner._get_pkgs_from_all_modules() - ) # pylint: disable=protected-access - assert len(pkgs) == 2 - assert "tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm" in pkgs - assert "tomcatjss-8.4.7-2.el8+1944+b6c8e16f.noarch.rpm" in pkgs diff --git a/tests/test_utils.py b/tests/test_utils.py index 0ba5433..8f788bc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from mock import MagicMock -from pubtools.pulplib import YumRepository +from pubtools.pulplib import YumRepository, RpmUnit from ubipop._utils import ( AssociateAction, AssociateActionModuleDefaults, @@ -11,7 +11,9 @@ UnassociateActionModuleDefaults, UnassociateActionModules, UnassociateActionRpms, + vercmp_sort, ) +from ubipop._matcher import UbiUnit def test_raise_not_implemented_pulp_action(): @@ -81,3 +83,41 @@ def test_get_action_unassociate(klass, method): assert "mock." + method in str(associate_action) assert current_units == units assert dst_repo_current.id == dst_repo.id + + +def test_vercmp_sort(): + """Tests all comparison methods for vercmp sort used for RPM packages comparison""" + vercmp_klass = vercmp_sort() + + unit_1 = vercmp_klass( + UbiUnit( + RpmUnit( + name="test", + version="10", + release="20", + epoch="1", + arch="x86_64", + ), + None, + ) + ) + + unit_2 = vercmp_klass( + UbiUnit( + RpmUnit( + name="test", + version="10", + release="200", + epoch="1", + arch="x86_64", + ), + None, + ) + ) + + assert (unit_1 < unit_2) is True + assert (unit_1 <= unit_2) is True + assert (unit_1 == unit_2) is False + assert (unit_1 >= unit_2) is False + assert (unit_1 > unit_2) is False + assert (unit_1 != unit_2) is True diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 87f28e3..2347277 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -1,3 +1,4 @@ +from datetime import date import logging import re @@ -20,7 +21,7 @@ UnassociateActionModuleDefaults, UnassociateActionRpms, ) -from ._matcher import ModularMatcher +from ._matcher import ModularMatcher, RpmMatcher _LOG = logging.getLogger("ubipop") @@ -378,151 +379,6 @@ def _match_module_defaults(self): if module_defaults: self.repos.module_defaults[fts[ft]].extend(module_defaults) - def _get_pkgs_from_all_modules(self): - modules = [] - for in_repo_rpm in self.repos.in_repos.rpm: - modules.extend(self.pulp.search_modules(in_repo_rpm)) - pkgs = set() - regex = r"\d+:" - reg = re.compile(regex) - for module in modules: - for pkg in module.packages: - rpm_without_epoch = reg.sub("", pkg) - rpm_filename = rpm_without_epoch + ".rpm" - pkgs.add(rpm_filename) - - return pkgs - - def _match_packages(self, input_repos, packages_dict): - """ - Add matching packages from whitelist - Globbing package name is not supported - """ - modular_pkgs = self._get_pkgs_from_all_modules() - fts = {} - for package_pattern in self.ubiconfig.packages.whitelist: - name = package_pattern.name - arch = None if package_pattern.arch in ("*", None) else package_pattern.arch - for repo in input_repos: - fts[ - (self._executor.submit(self.pulp.search_rpms, repo, name, arch)) - ] = name - - for ft in as_completed(fts): - packages = ft.result() - if packages: - for pkg in packages: - # skip modular packages, those are handled separately - if pkg.filename in modular_pkgs: - continue - - packages_dict[fts[ft]].append(pkg) - - def _match_binary_rpms(self): - self._match_packages(self.repos.in_repos.rpm, self.repos.packages) - - def _match_debug_rpms(self): - self._match_packages(self.repos.in_repos.debug, self.repos.debug_rpms) - - def _parse_blacklist_config(self): - packages_to_exclude = [] - for package_pattern in self.ubiconfig.packages.blacklist: - name_to_parse = package_pattern.name - globbing = "*" in name_to_parse - if globbing: - name = package_pattern.name[:-1] - else: - name = package_pattern.name - arch = None if package_pattern.arch in ("*", None) else package_pattern.arch - - packages_to_exclude.append((name, globbing, arch)) - - return packages_to_exclude - - def _exclude_blacklisted_packages(self): - blacklisted_binary = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.packages.values())) - ) - blacklisted_debug = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.debug_rpms.values())) - ) - - for pkg in blacklisted_binary: - # blacklist only non-modular pkgs - self.repos.packages[pkg.name][:] = [ - _pkg - for _pkg in self.repos.packages.get(pkg.name, []) - if _pkg.is_modular - ] - - # if there is nothing left, remove whole entry for package - if not self.repos.packages[pkg.name]: - self.repos.packages.pop(pkg.name, None) - - for pkg in blacklisted_debug: - # blacklist only non-modular debug pkgs - self.repos.debug_rpms[pkg.name][:] = [ - _pkg - for _pkg in self.repos.debug_rpms.get(pkg.name, []) - if _pkg.is_modular - ] - - # if there is nothing left, remove whole entry for debug package - if not self.repos.debug_rpms[pkg.name]: - self.repos.debug_rpms.pop(pkg.name, None) - - def _finalize_rpms_output_set(self): - for _, packages in self.repos.packages.items(): - self._finalize_output_units(packages, "rpm") - - def _finalize_debug_output_set(self): - for _, packages in self.repos.debug_rpms.items(): - self._finalize_output_units(packages, "rpm") - - def _finalize_output_units(self, units, type_id): - if type_id == "rpm": - self.sort_packages(units) - self.keep_n_latest_packages( - units - ) # with respect to packages referenced by modules - - def _create_srpms_output_set(self): - rpms = chain.from_iterable(self.repos.packages.values()) - binary_source_repo_map = {} - for package in rpms: - if package.sourcerpm is None: - _LOG.warning( - "Package %s doesn't reference its source rpm", package.name - ) - continue - in_repo = [ - r - for r in self.repos.in_repos.rpm - if r.id == package.associate_source_repo_id - ][0] - - if in_repo.id not in binary_source_repo_map: - binary_source_repo_map[in_repo.id] = in_repo.get_source_repository() - - associate_src_repo = binary_source_repo_map[in_repo.id] - - self.repos.source_rpms[package.name].append( - Package( - package.name, - package.sourcerpm, - is_modular=package.is_modular, - src_repo_id=associate_src_repo.id, - ) - ) - - blacklisted_srpms = self.get_blacklisted_packages( - list(chain.from_iterable(self.repos.source_rpms.values())) - ) - - for pkg in blacklisted_srpms: - if not pkg.is_modular: - self.repos.source_rpms.pop(pkg.name, None) - def _determine_pulp_actions(self, units, current, diff_f, extra_units=None): expected = list(units) if extra_units: @@ -546,9 +402,8 @@ def _get_pulp_actions_md_defaults(self, module_defaults, current): ) def _get_pulp_actions_pkgs(self, pkgs, current, modular_pkgs): - pkgs_list = list(chain.from_iterable(pkgs.values())) return self._determine_pulp_actions( - pkgs_list, current, self._diff_packages_by_filename, modular_pkgs + pkgs, current, self._diff_packages_by_filename, modular_pkgs ) def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): @@ -558,7 +413,7 @@ def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): """ uniq_srpms = {} - all_pkgs = list(chain.from_iterable(pkgs.values())) + list(modular) + all_pkgs = list(pkgs) + list(modular) # filter out packages that share same source rpm for pkg in all_pkgs: @@ -589,6 +444,7 @@ def _get_pulp_actions( Content that needs unassociation: unit is in current but not in expected No action: unit is in current and in expected """ + modules_assoc, modules_unassoc = self._get_pulp_actions_mds( self.repos.modules, current_modules_ft.result() ) @@ -668,19 +524,14 @@ def run_ubi_population(self): current_debug_rpms_ft, ) = self._get_current_content() - # start async querying for modulemds and modular packages + # start async querying for modulemds and modular and non-modular packages mm = ModularMatcher(self.repos.in_repos, self.ubiconfig.modules).run() - self.repos.modules = mm.modules - - self._match_binary_rpms() - if self.repos.out_repos.debug: - self._match_debug_rpms() - self._exclude_blacklisted_packages() + rm = RpmMatcher(self.repos.in_repos, self.ubiconfig).run() - # only non-modular packages - self._finalize_rpms_output_set() - self._finalize_debug_output_set() - self._create_srpms_output_set() + self.repos.modules = mm.modules + self.repos.packages = rm.binary_rpms + self.repos.debug_rpms = rm.debug_rpms + self.repos.source_rpms = rm.source_rpms self._match_module_defaults() @@ -856,60 +707,3 @@ def _publish_out_repos(self): if repo.result(): fts.append(repo.publish(options)) return fts - - def get_blacklisted_packages(self, package_list): - """ - Finds blacklisted packages in output sets - """ - blacklisted_pkgs = [] - for pattern_name, globbing, pattern_arch in self._parse_blacklist_config(): - for package in package_list: - name, _, _, _, arch = split_filename(package.filename) - blacklisted = False - if globbing: - if name.startswith(pattern_name): - blacklisted = True - else: - if name == pattern_name: - blacklisted = True - - if pattern_arch: - if arch != pattern_arch: - blacklisted = False - - if blacklisted: - blacklisted_pkgs.append(package) - - return blacklisted_pkgs - - def sort_packages(self, packages): - """ - Sort packages by vercmp - """ - packages.sort() - - def keep_n_latest_packages(self, packages, n=1): - """ - Keep n latest non-modular packages. - - Arguments: - packages (List[Package]): Sorted, oldest goes first - - Keyword arguments: - n (int): Number of non-modular package versions to keep - - Returns: - None. The packages list is changed in-place - """ - # Use a queue of n elements per arch - pkgs_per_arch = defaultdict(lambda: deque(maxlen=n)) - - for package in packages: - _, _, _, _, arch = split_filename(package.filename) - pkgs_per_arch[arch].append(package) - - latest_pkgs_per_arch = [ - pkg for pkg in chain.from_iterable(pkgs_per_arch.values()) - ] - - packages[:] = latest_pkgs_per_arch diff --git a/ubipop/_matcher.py b/ubipop/_matcher.py index 158612b..b985d69 100644 --- a/ubipop/_matcher.py +++ b/ubipop/_matcher.py @@ -1,12 +1,21 @@ import os +import logging +from itertools import chain +from collections import defaultdict, deque +from concurrent.futures import as_completed from pubtools.pulplib import Criteria +from pubtools.pulplib import Matcher as PulpLibMatcher from more_executors.futures import f_flat_map, f_return, f_sequence, f_proxy from more_executors import Executors -from ubipop._utils import split_filename - +from ubipop._utils import split_filename, vercmp_sort BATCH_SIZE = int(os.getenv("UBIPOP_BATCH_SIZE", "250")) +# need to set significantly lower batches for general rpm search +# otherwise db may very likely hit OOM error. +BATCH_SIZE_RPM = int(os.getenv("UBIPOP_BATCH_SIZE_RPM", "15")) + +_LOG = logging.getLogger("ubipop.matcher") class UbiUnit(object): @@ -45,6 +54,10 @@ def __init__(self, input_repos, ubi_config, workers=8): # we use executor from pulplib self._executor = Executors.thread_pool(max_workers=workers) + self.binary_rpms = None + self.debug_rpms = None + self.source_rpms = None + def run(self): """ This method needs to be implemented in subclasses and should @@ -53,11 +66,14 @@ def run(self): """ raise NotImplementedError - def _search_units(self, repo, criteria_list, content_type_id): + def _search_units( + self, repo, criteria_list, content_type_id, batch_size_override=None + ): """ Search for units of one content type associated with given repository by criteria. """ units = set() + batch_size = batch_size_override or BATCH_SIZE def handle_results(page): for unit in page.data: @@ -69,8 +85,8 @@ def handle_results(page): criteria_split = [] - for start in range(0, len(criteria_list), BATCH_SIZE): - criteria_split.append(criteria_list[start : start + BATCH_SIZE]) + for start in range(0, len(criteria_list), batch_size): + criteria_split.append(criteria_list[start : start + batch_size]) fts = [] for criteria_batch in criteria_split: @@ -99,40 +115,71 @@ def _create_or_criteria(self, fields, values): if len(val_tuple) != len(fields): raise ValueError for index, field in enumerate(fields): + inner_and_criteria.append(Criteria.with_field(field, val_tuple[index])) or_criteria.append(Criteria.and_(*inner_and_criteria)) return or_criteria - def _search_units_per_repos(self, or_criteria, repos, content_type): + def _search_units_per_repos( + self, or_criteria, repos, content_type, batch_size_override=None + ): units = [] for repo in repos: - units.append(self._search_units(repo, or_criteria, content_type)) + units.append( + self._search_units( + repo, + or_criteria, + content_type, + batch_size_override=batch_size_override, + ) + ) return f_proxy(f_flat_map(f_sequence(units), flatten_list_of_sets)) - def _search_rpms(self, or_criteria, repos): - return self._search_units_per_repos(or_criteria, repos, content_type="rpm") + def _search_rpms(self, or_criteria, repos, batch_size_override=None): + return self._search_units_per_repos( + or_criteria, + repos, + content_type="rpm", + batch_size_override=batch_size_override, + ) - def _search_srpms(self, or_criteria, repos): - return self._search_units_per_repos(or_criteria, repos, content_type="srpm") + def _search_srpms(self, or_criteria, repos, batch_size_override=None): + return self._search_units_per_repos( + or_criteria, + repos, + content_type="srpm", + batch_size_override=batch_size_override, + ) def _search_moludemds(self, or_criteria, repos): return self._search_units_per_repos(or_criteria, repos, content_type="modulemd") + def _get_srpms_criteria(self): + filenames = [] + for rpms_list in as_completed([self.binary_rpms, self.debug_rpms]): + for pkg in rpms_list: + if pkg.sourcerpm is None: + _LOG.warning( + "Package %s doesn't reference its source rpm", pkg.name + ) + continue + filenames.append((pkg.sourcerpm,)) + + pkgs_or_criteria = self._create_or_criteria(("filename",), filenames) + return pkgs_or_criteria + class ModularMatcher(Matcher): def __init__(self, input_repos, ubi_config): super(ModularMatcher, self).__init__(input_repos, ubi_config) self.modules = None - self.binary_rpms = None - self.debug_rpms = None - self.source_rpms = None def run(self): """Asynchronously creates criteria for pulp queries and - calls non-blocking search queries to pulp. + calls non-blocking search queries to pulp for ModularMatcher. Method immediately returns self, results of queries are stored as futures in public attributes of this class. Those can be accessed when they're needed. @@ -159,9 +206,7 @@ def run(self): self._search_rpms, rpms_criteria, self._input_repos.debug ) ) - srpms_criteria = f_proxy( - self._executor.submit(self._get_modular_srpms_criteria) - ) + srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) self.source_rpms = f_proxy( self._executor.submit( self._search_srpms, srpms_criteria, self._input_repos.source @@ -175,12 +220,6 @@ def _get_modular_rpms_criteria(self): pkgs_or_criteria = self._create_or_criteria(("filename",), filenames_to_search) return pkgs_or_criteria - def _get_modular_srpms_criteria(self): - non_source_pkg = list(self.binary_rpms) + list(self.debug_rpms) - filenames = [(pkg.sourcerpm,) for pkg in non_source_pkg] - pkgs_or_criteria = self._create_or_criteria(("filename",), filenames) - return pkgs_or_criteria - def _get_modulemds_criteria(self): criteria_values = [] for module in self._ubi_config: @@ -204,7 +243,7 @@ def _get_modulemd_output_set(self, modules): name_stream_modules_map.setdefault(key, []).append(modulemd) out = [] - # sort modulemds and keep N latest versions of them + # sort rpms and keep N latest versions of them for module_list in name_stream_modules_map.values(): module_list.sort(key=lambda module: module.version) self._keep_n_latest_modules(module_list) @@ -257,6 +296,182 @@ def _modular_rpms_filenames(self, modules): return filenames +class RpmMatcher(Matcher): + def __init__(self, input_repos, ubi_config): + super(RpmMatcher, self).__init__(input_repos, ubi_config) + + def run(self): + """Asynchronously creates criteria for pulp queries and + calls non-blocking search queries to pulp for RpmMatcher. + Method immediately returns self, results of queries are + stored as futures in public attributes of this class. Those + can be accessed when they're needed. + """ + # overriding the normal batch size for queries + # general queries for RPMs have extreme consumtion of RAM + # and can easily cause OOM on production-size databases + batch_size_override = BATCH_SIZE_RPM + + modular_rpm_filenames = f_proxy(self._get_pkgs_from_all_modules()) + rpms_criteria = f_proxy(self._executor.submit(self._get_rpms_criteria)) + + binary_rpms = f_proxy( + self._executor.submit( + self._search_rpms, + rpms_criteria, + self._input_repos.rpm, + batch_size_override, + ) + ) + + debug_rpms = f_proxy( + self._executor.submit( + self._search_rpms, + rpms_criteria, + self._input_repos.debug, + batch_size_override, + ) + ) + + self.binary_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, binary_rpms, modular_rpm_filenames + ) + ) + self.debug_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, debug_rpms, modular_rpm_filenames + ) + ) + + srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) + source_rpms = f_proxy( + self._executor.submit( + self._search_srpms, + srpms_criteria, + self._input_repos.source, + batch_size_override, + ) + ) + + self.source_rpms = f_proxy( + self._executor.submit( + self._get_rpm_output_set, source_rpms, modular_rpm_filenames + ) + ) + + return self + + def _get_rpms_criteria(self): + criteria_values = [] + + for package_pattern in self._ubi_config.packages.whitelist: + # skip src packages, they are searched seprately + if package_pattern.arch == "src": + continue + arch = ( + PulpLibMatcher.exists() + if package_pattern.arch in ("*", None) + else package_pattern.arch + ) + criteria_values.append((package_pattern.name, arch)) + + fields = ("name", "arch") + or_criteria = self._create_or_criteria(fields, criteria_values) + return or_criteria + + def _get_pkgs_from_all_modules(self): + # search for modulesmds in all input repos + # and extract filenames only + def extract_modular_filenames(): + modular_rpm_filenames = set() + for module in modules: + modular_rpm_filenames |= set(module.artifacts_filenames) + + return modular_rpm_filenames + + modules = self._search_moludemds([Criteria.true()], self._input_repos.rpm) + return self._executor.submit(extract_modular_filenames) + + def _get_rpm_output_set(self, rpms, modular_rpm_filenames): + blacklist_parsed = self._parse_blacklist_config() + name_rpms_maps = {} + + def is_blacklisted(rpm): + for name, globbing, arch in blacklist_parsed: + blacklisted = False + if globbing: + if rpm.name.startswith(name): + blacklisted = True + else: + if rpm.name == name: + blacklisted = True + if arch: + if rpm.arch != arch: + blacklisted = False + + if blacklisted: + return blacklisted + + for rpm in rpms: + # skip modular rpms + if rpm.filename in modular_rpm_filenames: + continue + # skip blacklisted rpms + if is_blacklisted(rpm): + continue + + name_rpms_maps.setdefault(rpm.name, []).append(rpm) + + out = [] + # sort rpms and keep N latest versions of them + for rpm_list in name_rpms_maps.values(): + rpm_list.sort(key=vercmp_sort()) + self._keep_n_latest_rpms(rpm_list) + out.extend(rpm_list) + + return out + + def _keep_n_latest_rpms(self, rpms, n=1): + """ + Keep n latest non-modular rpms. + + Arguments: + rpms (List[Rpm]): Sorted, oldest goes first + + Keyword arguments: + n (int): Number of non-modular package versions to keep + + Returns: + None. The packages list is changed in-place + """ + # Use a queue of n elements per arch + pkgs_per_arch = defaultdict(lambda: deque(maxlen=n)) + + for rpm in rpms: + pkgs_per_arch[rpm.arch].append(rpm) + + latest_pkgs_per_arch = [ + pkg for pkg in chain.from_iterable(pkgs_per_arch.values()) + ] + + rpms[:] = latest_pkgs_per_arch + + def _parse_blacklist_config(self): + packages_to_exclude = [] + for package_pattern in self._ubi_config.packages.blacklist: + globbing = package_pattern.name.endswith("*") + if globbing: + name = package_pattern.name[:-1] + else: + name = package_pattern.name + arch = None if package_pattern.arch in ("*", None) else package_pattern.arch + + packages_to_exclude.append((name, globbing, arch)) + + return packages_to_exclude + + def flatten_list_of_sets(list_of_sets): out = set() for one_set in list_of_sets: diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index 55c6b79..a635503 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -12,8 +12,6 @@ from urlparse import urljoin import requests -from rpm import labelCompare as label_compare # pylint: disable=no-name-in-module - _LOG = logging.getLogger("ubipop") @@ -299,27 +297,8 @@ def __init__( self.is_modular = is_modular # return name, ver, rel, epoch, arch _, self.version, self.release, self.epoch, _ = split_filename(self.filename) - self.evr_tuple = (self.epoch, self.version, self.release) self.associate_source_repo_id = src_repo_id - def __lt__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) < 0 - - def __gt__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) > 0 - - def __eq__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) == 0 - - def __le__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) <= 0 - - def __ge__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) >= 0 - - def __ne__(self, other): - return label_compare(self.evr_tuple, other.evr_tuple) != 0 - def __str__(self): return self.filename diff --git a/ubipop/_utils.py b/ubipop/_utils.py index 2a3d025..07f609f 100644 --- a/ubipop/_utils.py +++ b/ubipop/_utils.py @@ -1,3 +1,5 @@ +from rpm import labelCompare as label_compare # pylint: disable=no-name-in-module + # borrowed from https://github.com/rpm-software-management/yum def split_filename(filename): """ @@ -137,3 +139,29 @@ class UnassociateActionRpms(PulpAction): def get_actions(self, pulp_client_inst): return [(pulp_client_inst.unassociate_packages, self.dst_repo, self.units)] + + +def vercmp_sort(): + class Klass(object): + def __init__(self, package): + self.evr_tuple = (package.epoch, package.version, package.release) + + def __lt__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) < 0 + + def __gt__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) > 0 + + def __eq__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) == 0 + + def __le__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) <= 0 + + def __ge__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) >= 0 + + def __ne__(self, other): + return label_compare(self.evr_tuple, other.evr_tuple) != 0 + + return Klass