diff --git a/requirements.txt b/requirements.txt index af035fa..c9ffd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ more-executors>2.1.0 ubi-config>=2.2.0 rpm-py-installer pyrsistent<0.17; python_version < '3' -pubtools-pulplib>=2.9.0 \ No newline at end of file +pubtools-pulplib>=2.9.0 +attrs \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..48aa0f6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,32 @@ +import pytest + +from pubtools.pulplib import RpmUnit, ModulemdUnit, ModulemdDefaultsUnit, FakeController +from ubipop._matcher import UbiUnit + + +def _get_test_unit(klass, **kwargs): + repo_id = kwargs.pop("src_repo_id") + return UbiUnit(klass(**kwargs), repo_id) + + +def get_rpm_unit(**kwargs): + return _get_test_unit(RpmUnit, **kwargs) + + +def get_srpm_unit(**kwargs): + kwargs["content_type_id"] = "srpm" + kwargs["arch"] = "src" + return _get_test_unit(RpmUnit, **kwargs) + + +def get_modulemd_unit(**kwargs): + return _get_test_unit(ModulemdUnit, **kwargs) + + +def get_modulemd_defaults_unit(**kwargs): + return _get_test_unit(ModulemdDefaultsUnit, **kwargs) + + +@pytest.fixture(name="pulp") +def fake_pulp(): + yield FakeController() diff --git a/tests/test_matcher.py b/tests/test_matcher.py index c811e99..7367829 100644 --- a/tests/test_matcher.py +++ b/tests/test_matcher.py @@ -6,8 +6,8 @@ RpmUnit, Criteria, YumRepository, - FakeController, ModulemdUnit, + ModulemdDefaultsUnit, ) from ubiconfig import UbiConfig @@ -22,11 +22,6 @@ from ubipop._utils import vercmp_sort -@pytest.fixture(name="pulp") -def fake_pulp(): - yield FakeController() - - @pytest.fixture(name="ubi_config") def fake_ubi_config(): config_dict = { @@ -206,7 +201,7 @@ def test_search_rpms(pulp): matcher = Matcher(None, None) criteria = matcher._create_or_criteria(["filename"], [("test.x86_64.rpm",)]) # let Future return result - result = matcher._search_rpms(criteria, [repo]).result() + result = matcher.search_rpms(criteria, [repo]).result() # there should be be only one unit in the result set according to criteria assert len(result) == 1 assert result.pop().filename == "test.x86_64.rpm" @@ -241,7 +236,7 @@ def test_search_srpms(pulp): matcher = Matcher(None, None) criteria = matcher._create_or_criteria(["filename"], [("test.src.rpm",)]) # let Future return result - result = matcher._search_srpms(criteria, [repo]).result() + result = matcher.search_srpms(criteria, [repo]).result() # there should be be only one unit in the result set according to criteria assert len(result) == 1 assert result.pop().filename == "test.src.rpm" @@ -274,12 +269,43 @@ def test_search_moludemds(pulp): matcher = Matcher(None, None) criteria = matcher._create_or_criteria(["name", "stream"], [("test", "10")]) # let Future return result - result = matcher._search_moludemds(criteria, [repo]).result() + result = matcher.search_modulemds(criteria, [repo]).result() # there should be be only one unit in the result set according to criteria assert len(result) == 1 assert result.pop().nsvca == "test:10:100:abcdef:x86_64" +def test_search_moludemd_defaults(pulp): + """Test convenient method for searching modulemd_defaults""" + repo = YumRepository( + id="test_repo_1", + ) + repo.__dict__["_client"] = pulp.client + unit_1 = ModulemdDefaultsUnit( + name="test", + stream="10", + repo_id="test_repo_1", + ) + unit_2 = ModulemdDefaultsUnit( + name="test", + stream="20", + repo_id="test_repo_1", + ) + + pulp.insert_repository(repo) + pulp.insert_units(repo, [unit_1, unit_2]) + + matcher = Matcher(None, None) + criteria = matcher._create_or_criteria(["name", "stream"], [("test", "10")]) + # let Future return result + result = matcher.search_modulemd_defaults(criteria, [repo]).result() + # there should be be only one unit in the result set according to criteria + assert len(result) == 1 + found_unit = result.pop() + assert found_unit.name == "test" + assert found_unit.stream == "10" + + def test_modular_rpms_filenames(ubi_config): """Test getting filename from module artifacts, srpms are skipped.""" matcher = ModularMatcher(None, ubi_config.modules) @@ -490,6 +516,25 @@ def test_get_modulemds_criteria(ubi_config): # let's not test internal structure of criteria, that's responsibility of pulplib +def test_get_modulemd_defaults_criteria(): + """Test proper creation of criteria for modulemd_defaults query""" + matcher = ModularMatcher(None, None) + unit = UbiUnit( + ModulemdUnit( + name="test", stream="10", version=100, context="abcd", arch="x86_64" + ), + None, + ) + matcher.modules = [unit] + criteria = matcher._get_modulemd_defaults_criteria() + # there should be 1 criterium created based on modules list of Matcher obj. + assert len(criteria) == 1 + # it should be instance of Criteria + for crit in criteria: + assert isinstance(crit, Criteria) + # let's not test internal structure of criteria, that's responsibility of pulplib + + def test_get_modular_srpms_criteria(ubi_config): """Testing creation of criteria for srpms query""" matcher = ModularMatcher(None, ubi_config.modules) @@ -614,10 +659,16 @@ def test_modular_matcher_run(pulp, ubi_config): "test-7:1.0-1.x86_64.src", ], ) + + modulemd_defaults = ModulemdDefaultsUnit( + name="fake_name", + stream="fake_stream", + repo_id="binary_repo", + ) pulp.insert_repository(repo_1) pulp.insert_repository(repo_2) pulp.insert_repository(repo_3) - pulp.insert_units(repo_1, [unit_1, modulemd]) + pulp.insert_units(repo_1, [unit_1, modulemd, modulemd_defaults]) pulp.insert_units(repo_2, [unit_2]) pulp.insert_units(repo_3, [unit_3]) @@ -627,6 +678,7 @@ def test_modular_matcher_run(pulp, ubi_config): # each public attribute is properly set with one unit assert len(matcher.modules) == 1 + assert len(matcher.modulemd_defaults) == 1 assert len(matcher.binary_rpms) == 1 assert len(matcher.debug_rpms) == 1 assert len(matcher.source_rpms) == 1 @@ -636,6 +688,10 @@ def test_modular_matcher_run(pulp, ubi_config): assert output_module.nsvca == "fake_name:fake_stream:100:abcd:x86_64" assert output_module.associate_source_repo_id == "binary_repo" + output_modulemd_defaults = matcher.modulemd_defaults.pop() + assert output_modulemd_defaults.name == "fake_name" + assert output_modulemd_defaults.stream == "fake_stream" + rpm = matcher.binary_rpms.pop() assert rpm.filename == "test-1.0-1.x86_64.x86_64.rpm" assert rpm.associate_source_repo_id == "binary_repo" diff --git a/tests/test_pulp.py b/tests/test_pulp.py index 13010a6..e3ee751 100644 --- a/tests/test_pulp.py +++ b/tests/test_pulp.py @@ -16,7 +16,12 @@ from pubtools.pulplib import YumRepository from ubipop import _pulp_client as pulp_client -from ubipop._pulp_client import Package, ModuleDefaults, Pulp, PulpRetryAdapter +from ubipop._pulp_client import Pulp, PulpRetryAdapter + +from .conftest import ( + get_rpm_unit, + get_modulemd_defaults_unit, +) ORIG_HTTP_TOTAL_RETRIES = pulp_client.HTTP_TOTAL_RETRIES ORIG_HTTP_RETRY_BACKOFF = pulp_client.HTTP_RETRY_BACKOFF @@ -51,12 +56,25 @@ def fixture_mock_repo(): @pytest.fixture(name="mock_package") def fixture_mock_package(): - yield Package("foo-pkg", "foo-pkg.rpm", "src_repo_id") + yield get_rpm_unit( + name="foo-pkg", + version="10", + release="1", + arch="x86_64", + filename="foo-pkg.rpm", + src_repo_id="src_repo_id", + ) @pytest.fixture(name="mock_mdd") def fixture_mock_mdd(): - yield ModuleDefaults("virt", "rhel", {"2.6": ["common"]}, "src_repo_id") + yield get_modulemd_defaults_unit( + name="virt", + stream="rhel", + profiles={"2.6": ["common"]}, + repo_id="src_repo_id", + src_repo_id="src_repo_id", + ) @pytest.fixture(name="mock_response_for_async_req") @@ -64,90 +82,6 @@ def fixture_mock_response_for_async_req(): yield {"spawned_tasks": [{"task_id": "foo_task_id"}]} -@pytest.fixture(name="search_rpms_response") -def fixture_search_rpms_response(request): - try: - pkg_number = request.param - except AttributeError: - pkg_number = 1 - - yield [ - { - "metadata": { - "name": "foo-pkg", - "filename": "foo-pkg.rpm", - "sourcerpm": "foo-pkg.src.rpm", - "is_modular": False, - } - } - for _ in range(pkg_number) - ] - - -@pytest.fixture(name="search_modules_response") -def fixture_search_modules_response(): - yield [ - { - "metadata": { - "name": "foo-module", - "stream": "9.6", - "version": 1111, - "context": "foo-context", - "arch": "x86_64", - "artifacts": ["foo-pkg"], - "profiles": {"foo-prof": ["pkg-name"]}, - }, - } - ] - - -@pytest.fixture(name="search_module_defaults_response") -def fixture_search_module_defaults_response(): - yield [ - { - "metadata": { - "name": "virt", - "stream": "rhel", - "profiles": {"rhel": ["default", "common"]}, - }, - } - ] - - -@pytest.fixture(name="mock_search_rpms") -def fixture_mock_search_rpms(requests_mock, mock_repo, search_rpms_response): - url = "/pulp/api/v2/repositories/{REPO_ID}/search/units/".format( - REPO_ID=mock_repo.id - ) - requests_mock.register_uri("POST", url, json=search_rpms_response) - - -@pytest.fixture(name="mock_search_modules") -def fixture_mock_search_modules(requests_mock, mock_repo, search_modules_response): - url = "/pulp/api/v2/repositories/{REPO_ID}/search/units/".format( - REPO_ID=mock_repo.id - ) - requests_mock.register_uri("POST", url, json=search_modules_response) - - -@pytest.fixture(name="mock_search_module_defaults") -def fixture_mock_search_module_defaults( - requests_mock, mock_repo, search_module_defaults_response -): - url = "/pulp/api/v2/repositories/{REPO_ID}/search/units/".format( - REPO_ID=mock_repo.id - ) - requests_mock.register_uri("POST", url, json=search_module_defaults_response) - - -@pytest.fixture(name="mock_publish") -def fixture_mock_publish(requests_mock, mock_repo, mock_response_for_async_req): - url = "/pulp/api/v2/repositories/{repo_id}/actions/publish/".format( - repo_id=mock_repo.id - ) - requests_mock.register_uri("POST", url, json=mock_response_for_async_req) - - @pytest.fixture(name="mock_associate") def fixture_mock_associate(requests_mock, mock_repo, mock_response_for_async_req): url = "/pulp/api/v2/repositories/{dst_repo}/actions/associate/".format( @@ -192,46 +126,6 @@ def test_unassociate_module_defaults(mock_pulp, mock_unassociate, mock_repo, moc assert task_ids[0] == "foo_task_id" -def test_search_rpms(mock_pulp, mock_search_rpms, mock_repo): - # pylint: disable=unused-argument - found_rpms = mock_pulp.search_rpms(mock_repo) - assert len(found_rpms) == 1 - assert found_rpms[0].name == "foo-pkg" - assert found_rpms[0].filename == "foo-pkg.rpm" - assert found_rpms[0].sourcerpm == "foo-pkg.src.rpm" - assert found_rpms[0].is_modular is False - - -@pytest.mark.parametrize("search_rpms_response", [0, 1, 2], indirect=True) -def test_search_rpms_by_filename( - mock_pulp, mock_search_rpms, mock_repo, search_rpms_response -): - # pylint: disable=unused-argument - found_rpms = mock_pulp.search_rpms(mock_repo, filename="foo-pkg.rpm") - assert len(search_rpms_response) == len(found_rpms) - - -def test_search_modules(mock_pulp, mock_search_modules, mock_repo): - # pylint: disable=unused-argument - found_modules = mock_pulp.search_modules(mock_repo) - assert len(found_modules) == 1 - - assert found_modules[0].nsvca == "foo-module:9.6:1111:foo-context:x86_64" - assert found_modules[0].packages == ["foo-pkg"] - assert found_modules[0].profiles == {"foo-prof": ["pkg-name"]} - - -def test_search_module_defaults(mock_pulp, mock_search_module_defaults, mock_repo): - # pylint: disable=unused-argument - found_module_defaults = mock_pulp.search_module_defaults( - mock_repo, name="virt", stream="rhel" - ) - assert len(found_module_defaults) == 1 - assert found_module_defaults[0].name == "virt" - assert found_module_defaults[0].stream == "rhel" - assert found_module_defaults[0].name_profiles == "virt:[rhel:common,default]" - - @pytest.fixture(name="search_task_response") def fixture_search_task_response(): yield {"state": "finished", "task_id": "test_task"} @@ -276,24 +170,6 @@ def make_mock_response(status, text): "should_retry,err_status_code,env_retries,retry_call,retry_args,ok_response,expected_retries", [ # test everything is retryable - ( - True, - 500, - None, - "search_rpms", - (MagicMock(id="fake_id"),), - "[]", - pulp_client.HTTP_TOTAL_RETRIES, - ), - ( - True, - 500, - None, - "search_modules", - (MagicMock(id="fake_id"),), - "[]", - pulp_client.HTTP_TOTAL_RETRIES, - ), ( True, 500, @@ -336,9 +212,35 @@ def make_mock_response(status, text): pulp_client.HTTP_TOTAL_RETRIES, ), # test custom number of retries - (True, 500, 3, "search_rpms", (MagicMock(id="fake_id"),), "{}", 3), + ( + True, + 500, + 3, + "associate_units", + ( + MagicMock(id="fake_id"), + MagicMock(id="fake_id"), + MagicMock(), + ["rpm"], + ), + '{"spawned_tasks":[]}', + 3, + ), # test 400 status is not retryable - (False, 400, 3, "search_rpms", (MagicMock(id="fake_id"),), "{}", 3), + ( + False, + 400, + 3, + "associate_units", + ( + MagicMock(id="fake_id"), + MagicMock(id="fake_id"), + MagicMock(), + ["rpm"], + ), + '{"spawned_tasks":[]}', + 3, + ), ], ) def test_retries( diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 916bad2..ec03e3d 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -6,6 +6,7 @@ import shutil import sys import tempfile + import pytest import ubiconfig @@ -16,11 +17,13 @@ Distributor, ModulemdUnit, RpmUnit, + ModulemdDefaultsUnit, ) from mock import MagicMock, patch, call from more_executors import Executors from more_executors.futures import f_proxy, f_return from ubipop import ( + RepoContent, UbiPopulateRunner, UbiRepoSet, RepoSet, @@ -29,16 +32,18 @@ RepoMissing, PopulationSourceMissing, ) -from ubipop._pulp_client import Module, ModuleDefaults, Package from ubipop._utils import ( AssociateActionModules, UnassociateActionModules, AssociateActionModuleDefaults, UnassociateActionModuleDefaults, - split_filename, ) -from ubipop._matcher import UbiUnit - +from .conftest import ( + get_rpm_unit, + get_srpm_unit, + get_modulemd_unit, + get_modulemd_defaults_unit, +) TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "./data") @@ -150,53 +155,6 @@ def get_test_repo(**kwargs): ) -def get_test_pkg(**kwargs): - return Package( - kwargs.get("name"), - kwargs.get("filename"), - kwargs.get("src_repo_id"), - sourcerpm_filename=kwargs.get("sourcerpm_filename"), - is_modular=kwargs.get("is_modular", False), - ) - - -def get_test_mod(**kwargs): - if kwargs.get("pulplib"): - unit = UbiUnit( - ModulemdUnit( - name=kwargs.get("name", ""), - stream=kwargs.get("stream", ""), - version=kwargs.get("version", 0), - context=kwargs.get("context", ""), - arch=kwargs.get("arch", ""), - artifacts=kwargs.get("packages", []), - profiles=kwargs.get("profiles", {}), - ), - kwargs.get("src_repo_id"), - ) - else: - unit = Module( - kwargs.get("name", ""), - kwargs.get("stream", ""), - kwargs.get("version", 0), - kwargs.get("context", ""), - kwargs.get("arch", ""), - kwargs.get("packages", []), - kwargs.get("profiles", {}), - kwargs.get("src_repo_id"), - ) - return unit - - -def get_test_mod_defaults(**kwargs): - return ModuleDefaults( - kwargs["name"], - kwargs["stream"], - kwargs["profiles"], - kwargs.get("src_repo_id"), - ) - - def test_get_output_repo_ids(ubi_repo_set): repo_ids = ubi_repo_set.get_output_repo_ids() assert repo_ids == set(["ubi-foo-rpms", "ubi-foo-source", "ubi-foo-debug"]) @@ -354,73 +312,58 @@ def test_get_ubi_repo_sets(get_debug_repository, get_source_repository): assert output_repos.debug.id == "ubi_debug" -def _get_search_rpms_side_effect(package_name_or_filename_or_list, debug_only=False): - def _f(*args, **kwargs): - if debug_only and "debug" not in args[0].id: - return - - if len(args) > 1 and args[1] == package_name_or_filename_or_list: - return [get_test_pkg(name=args[1], filename=args[1] + ".rpm")] - - if isinstance(package_name_or_filename_or_list, list): - if kwargs["filename"] in package_name_or_filename_or_list: - return [ - get_test_pkg( - name=split_filename(kwargs["filename"])[0], - filename=kwargs["filename"], - ), - ] - - if ( - "filename" in kwargs - and package_name_or_filename_or_list == kwargs["filename"] - ): - return [ - get_test_pkg( - name=split_filename(kwargs["filename"])[0], - filename=kwargs["filename"], - ), - ] - - return _f - - -def test_match_module_defaults(mock_ubipop_runner): - unit = UbiUnit( - ModulemdUnit( - name="virt", - stream="rhel", +def test_diff_modules(mock_ubipop_runner): + curr = [ + get_modulemd_unit( + name="1", + stream="foo", version=1, - context="abcd", + context="bar", arch="x86_64", - profiles={"2.5": ["common"]}, + src_repo_id="fake-repo", + ), + get_modulemd_unit( + name="2", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="fake-repo", + ), + get_modulemd_unit( + name="3", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="fake-repo", ), - None, - ) - mock_ubipop_runner.repos.modules = f_proxy(f_return(set([unit]))) - mock_ubipop_runner.pulp.search_module_defaults.return_value = [ - get_test_mod_defaults(name="virt", stream="rhel", profiles={"2.5": ["common"]}), - ] - - mock_ubipop_runner._match_module_defaults() # pylint: disable=protected-access - - assert len(mock_ubipop_runner.repos.module_defaults) == 1 - md_d = mock_ubipop_runner.repos.module_defaults["virtrhel"] - assert len(md_d) == 1 - assert md_d[0].name == "virt" - assert md_d[0].name_profiles == "virt:[2.5:common]" - - -def test_diff_modules(mock_ubipop_runner): - curr = [ - get_test_mod(name="1"), - get_test_mod(name="2"), - get_test_mod(name="3"), ] expected = [ - get_test_mod(name="2", pulplib=True), - get_test_mod(name="3", pulplib=True), - get_test_mod(name="4", pulplib=True), + get_modulemd_unit( + name="2", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="fake-repo", + ), + get_modulemd_unit( + name="3", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="fake-repo", + ), + get_modulemd_unit( + name="4", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="fake-repo", + ), ] diff = mock_ubipop_runner._diff_modules_by_nsvca( @@ -680,106 +623,131 @@ def test_create_output_file_all_repos( shutil.rmtree(path) -@pytest.fixture(name="mock_current_content_ft") -def fixture_mock_current_content_ft(): - current_modules_ft = MagicMock() - current_rpms_ft = MagicMock() - current_srpms_ft = MagicMock() - current_debug_rpms_ft = MagicMock() - current_module_default_ft = MagicMock() +@pytest.fixture(name="mock_current_content") +def fixture_mock_current_content(): + rpm = get_rpm_unit( + name="rpm_current", + filename="rpm_current.rpm", + version="1", + release="0", + arch="x86_64", + src_repo_id="ubi-foo-rpms", + ) - current_modules_ft.result.return_value = [ - get_test_mod(name="md_current"), - ] - current_module_default_ft.result.return_value = [ - get_test_mod_defaults( - name="mdd_current", stream="rhel", profiles={"2.5": "common"} - ), - ] - current_rpms_ft.result.return_value = [ - get_test_pkg(name="rpm_current", filename="rpm_current.rpm"), - ] - current_srpms_ft.result.return_value = [ - get_test_pkg(name="srpm_current", filename="srpm_current.src.rpm"), - ] - current_debug_rpms_ft.result.return_value = [ - get_test_pkg(name="debug_rpm_current", filename="debug_rpm_current.rpm"), - ] + srpm = get_srpm_unit( + name="srpm_current", + filename="srpm_current.src.rpm", + version="1", + release="0", + arch="x86_64", + src_repo_id="ubi-foo-source", + ) - yield current_modules_ft, current_module_default_ft, current_rpms_ft, current_srpms_ft, current_debug_rpms_ft + debug_rpm = get_rpm_unit( + name="debug_rpm_current", + filename="debug_rpm_current.rpm", + version="1", + release="0", + arch="x86_64", + src_repo_id="ubi-foo-debug", + ) + modulemd_unit = get_modulemd_unit( + name="md_current", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="ubi-foo-rpms", + ) + modulemd_defaults_unit = get_modulemd_defaults_unit( + name="mdd_current", + stream="rhel", + profiles={"2.5": ["common"]}, + repo_id="ubi-foo-rpms", + src_repo_id="ubi-foo-rpms", + ) -def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): - mock_ubipop_runner.repos.modules = f_proxy( - f_return(set([get_test_mod(name="test_md", pulplib=True)])) + binary_rpms = f_proxy(f_return([rpm])) + debug_rpms = f_proxy(f_return([srpm])) + source_rpms = f_proxy(f_return([debug_rpm])) + modulemds = f_proxy(f_return([modulemd_unit])) + modulemd_defaults = f_proxy(f_return([modulemd_defaults_unit])) + + repo_content = RepoContent( + binary_rpms, debug_rpms, source_rpms, modulemds, modulemd_defaults ) - mock_ubipop_runner.repos.module_defaults = { - "test": [ - get_test_mod_defaults( - name="test_mdd", stream="rhel", profiles={"2.5": "uncommon"} - ), - ], - } - - binary_rpms = [ - UbiUnit( - RpmUnit( - name="test_rpm", - version="1", - release="2", - arch="x86_64", - filename="test_rpm.rpm", - ), - "foo-rpms", - ) - ] - debug_rpms = [ - UbiUnit( - RpmUnit( - name="test_debug_pkg", - version="1", - release="2", - arch="x86_64", - filename="test_rpm.rpm", - ), - "foo-debug", - ) - ] - source_rpms = [ - UbiUnit( - RpmUnit( - name="test_srpm", - version="1", - release="2", - arch="x86_64", - filename="test_srpm.src.rpm", - ), - "foo-source", - ) - ] + yield repo_content - mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) - mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) - modular_binary = UbiUnit( - RpmUnit(name="modular_binary", version="1.0", release="1", arch="x86_64"), - "foo-rpms", +def test_get_pulp_actions(mock_ubipop_runner, mock_current_content): + binary_rpm = get_rpm_unit( + name="test_rpm", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + src_repo_id="foo-rpms", ) - modular_debug = UbiUnit( - RpmUnit(name="modular_debug", version="1.0", release="1", arch="x86_64"), - "foo-debug", + + debug_rpm = get_rpm_unit( + name="test_debug_pkg", + version="1", + release="2", + arch="x86_64", + filename="test_rpm.rpm", + src_repo_id="foo-debug", ) - modular_source = UbiUnit( - RpmUnit( - name="modular_source", - version="1.0", - release="1", - arch="src", - content_type_id="srpm", - ), - "foo-source", + + source_rpm = get_srpm_unit( + name="test_srpm", + version="1", + release="2", + filename="test_srpm.src.rpm", + src_repo_id="foo-source", + ) + modulemd = get_modulemd_unit( + name="test_md", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="foo-rpms", + ) + modulemd_defaults = get_modulemd_defaults_unit( + name="test_mdd", + stream="rhel", + profiles={"2.5": ["uncommon"]}, + repo_id="foo-rpms", + src_repo_id="foo-rpms", + ) + + mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return([source_rpm])) + mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) + mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) + + modular_binary = get_rpm_unit( + name="modular_binary", + version="1.0", + release="1", + arch="x86_64", + src_repo_id="foo-rpms", + ) + modular_debug = get_rpm_unit( + name="modular_debug", + version="1.0", + release="1", + arch="x86_64", + src_repo_id="foo-debug", + ) + modular_source = get_srpm_unit( + name="modular_source", + version="1.0", + release="1", + src_repo_id="foo-source", ) # pylint: disable=protected-access @@ -789,10 +757,10 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): mdd_association, mdd_unassociation, ) = mock_ubipop_runner._get_pulp_actions( - *mock_current_content_ft, + mock_current_content, modular_binary=f_proxy(f_return(set([modular_binary]))), modular_debug=f_proxy(f_return(set([modular_debug]))), - modular_source=f_proxy(f_return(set([modular_source]))) + modular_source=f_proxy(f_return(set([modular_source]))), ) # firstly, check correct associations, there should 1 unit of each type associated @@ -868,58 +836,55 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft): assert mdd_unassociation.dst_repo.id == "ubi-foo-rpms" -def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft): - mock_ubipop_runner.repos.modules = f_proxy( - f_return(set([get_test_mod(name="md_current", pulplib=True)])) +def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content): + binary_rpm = get_rpm_unit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + src_repo_id="foo-rpms", ) - mock_ubipop_runner.repos.module_defaults = { - "test": [ - get_test_mod_defaults( - name="mdd_current", stream="rhel", profiles={"2.5": "common"} - ), - ], - } - - binary_rpms = [ - UbiUnit( - RpmUnit( - name="rpm_current", - version="1", - release="2", - arch="x86_64", - filename="rpm_current.rpm", - ), - "foo-rpms", - ) - ] - debug_rpms = [ - UbiUnit( - RpmUnit( - name="debug_rpm_current", - version="1", - release="2", - arch="x86_64", - filename="debug_rpm_current.rpm", - ), - "foo-debug", - ) - ] - source_rpms = [ - UbiUnit( - RpmUnit( - name="srpm_current", - version="1", - release="2", - arch="x86_64", - filename="srpm_current.src.rpm", - ), - "foo-source", - ) - ] - mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) - mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) + debug_rpm = get_rpm_unit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + src_repo_id="foo-debug", + ) + + source_rpm = get_srpm_unit( + name="srpm_current", + version="1", + release="2", + filename="srpm_current.src.rpm", + src_repo_id="foo-source", + ) + + modulemd = get_modulemd_unit( + name="md_current", + stream="foo", + version=1, + context="bar", + arch="x86_64", + src_repo_id="ubi-foo-rpms", + ) + + modulemd_defaults = get_modulemd_defaults_unit( + name="mdd_current", + stream="rhel", + profiles={"2.5": ["common"]}, + repo_id="foo-rpms", + src_repo_id="foo-rpms", + ) + + mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) + mock_ubipop_runner.repos.source_rpms = f_proxy(f_return([source_rpm])) + mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) + mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) # pylint: disable=protected-access ( @@ -928,7 +893,7 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft mdd_association, mdd_unassociation, ) = mock_ubipop_runner._get_pulp_actions( - *mock_current_content_ft, modular_binary=[], modular_debug=[], modular_source=[] + mock_current_content, modular_binary=[], modular_debug=[], modular_source=[] ) # firstly, check correct associations, there should 0 units associated @@ -960,18 +925,31 @@ def test_log_pulp_action(capsys, set_logging, mock_ubipop_runner): set_logging.addHandler(logging.StreamHandler(sys.stdout)) src_repo = get_test_repo(id="test_src") dst_repo = get_test_repo(id="test_dst") + + unit_1 = get_modulemd_unit( + name="test_assoc", + stream="fake-stream", + version=1, + context="fake-context", + arch="x86_64", + src_repo_id=src_repo.id, + ) + unit_2 = get_modulemd_unit( + name="test_unassoc", + stream="fake-stream", + version=1, + context="fake-context", + arch="x86_64", + src_repo_id=src_repo.id, + ) associations = [ AssociateActionModules( - [get_test_mod(name="test_assoc", src_repo_id=src_repo.id, pulplib=True)], + [unit_1], dst_repo, [src_repo], ) ] - unassociations = [ - UnassociateActionModules( - [get_test_mod(name="test_unassoc", pulplib=True)], dst_repo - ) - ] + unassociations = [UnassociateActionModules([unit_2], dst_repo)] mock_ubipop_runner.log_pulp_actions(associations, unassociations) out, err = capsys.readouterr() @@ -980,11 +958,11 @@ def test_log_pulp_action(capsys, set_logging, mock_ubipop_runner): assert err == "" assert ( assoc_line.strip() - == "Would associate ModulemdUnit(name='test_assoc', stream='', version=0, context='', arch='', content_type_id='modulemd', repository_memberships=None, artifacts=[], profiles={}) from test_src to test_dst" + == "Would associate ModulemdUnit(name='test_assoc', stream='fake-stream', version=1, context='fake-context', arch='x86_64', content_type_id='modulemd', repository_memberships=None, artifacts=None, profiles=None) from test_src to test_dst" ) assert ( unassoc_line.strip() - == "Would unassociate ModulemdUnit(name='test_unassoc', stream='', version=0, context='', arch='', content_type_id='modulemd', repository_memberships=None, artifacts=[], profiles={}) from test_dst" + == "Would unassociate ModulemdUnit(name='test_unassoc', stream='fake-stream', version=1, context='fake-context', arch='x86_64', content_type_id='modulemd', repository_memberships=None, artifacts=None, profiles=None) from test_dst" ) @@ -1007,113 +985,93 @@ def test_log_pulp_action_no_actions(capsys, set_logging, mock_ubipop_runner): assert unassoc_line.strip() == "No unassociation expected for modules from test_dst" -def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft): +def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content): + binary_rpm = get_rpm_unit( + name="rpm_current", + version="1", + release="2", + arch="x86_64", + filename="rpm_current.rpm", + src_repo_id="foo-rpms", + ) - mock_ubipop_runner.repos.modules = f_proxy( - f_return(set([get_test_mod(name="md_current", pulplib=True)])) + debug_rpm = get_rpm_unit( + name="debug_rpm_current", + version="1", + release="2", + arch="x86_64", + filename="debug_rpm_current.rpm", + src_repo_id="foo-debug", ) - mock_ubipop_runner.repos.module_defaults = { - "test": [ - get_test_mod_defaults( - name="mdd_current", stream="rhel", profiles={"2.5": "common"} - ) - ] - } - - binary_rpms = [ - UbiUnit( - RpmUnit( - name="rpm_current", - version="1", - release="2", - arch="x86_64", - filename="rpm_current.rpm", - ), - "foo-rpms", - ) - ] - debug_rpms = [ - UbiUnit( - RpmUnit( - name="debug_rpm_current", - version="1", - release="2", - arch="x86_64", - filename="debug_rpm_current.rpm", - ), - "foo-debug", - ) - ] + source_rpms = [ - UbiUnit( - RpmUnit( - name="test_srpm", - version="1.0", - release="1", - arch="src", - filename="test_srpm-1.0-1.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="test_srpm", + version="1.0", + release="1", + filename="test_srpm-1.0-1.src.rpm", + src_repo_id="foo-source", ), - UbiUnit( - RpmUnit( - name="test_srpm", - version="1.0", - release="2", - arch="src", - filename="test_srpm-1.0-2.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="test_srpm", + version="1.0", + release="2", + filename="test_srpm-1.0-2.src.rpm", + src_repo_id="foo-source", ), - UbiUnit( - RpmUnit( - name="test_srpm", - version="1.0", - release="2", - arch="src", - filename="test_srpm-1.1-1.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="test_srpm", + version="1.0", + release="2", + filename="test_srpm-1.1-1.src.rpm", + src_repo_id="foo-source", ), - UbiUnit( - RpmUnit( - name="test_pkg", - version="1", - release="2", - arch="src", - filename="srpm_new.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="test_pkg", + version="1", + release="2", + filename="srpm_new.src.rpm", + src_repo_id="foo-source", ), - UbiUnit( - RpmUnit( - name="foo_pkg", - version="1", - release="2", - arch="src", - filename="srpm_new.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="foo_pkg", + version="1", + release="2", + filename="srpm_new.src.rpm", + src_repo_id="foo-source", ), - UbiUnit( - RpmUnit( - name="bar_pkg", - version="1", - release="2", - arch="src", - filename="srpm_new_next.src.rpm", - ), - "foo-source", + get_srpm_unit( + name="bar_pkg", + version="1", + release="2", + filename="srpm_new_next.src.rpm", + src_repo_id="foo-source", ), ] - - mock_ubipop_runner.repos.packages = f_proxy(f_return(binary_rpms)) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return(debug_rpms)) + modulemd = get_modulemd_unit( + name="md_current", + stream="fake-stream", + version=1, + context="fake-context", + arch="x86_64", + src_repo_id="foo-rpms", + ) + modulemd_defaults = get_modulemd_defaults_unit( + name="mdd_current", + stream="rhel", + profiles={"2.5": ["common"]}, + repo_id="foo-rpms", + src_repo_id="foo-rpms", + ) + mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) + mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) mock_ubipop_runner.repos.source_rpms = f_proxy(f_return(source_rpms)) + mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) + mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) # pylint: disable=protected-access associations, _, _, _ = mock_ubipop_runner._get_pulp_actions( - *mock_current_content_ft, modular_binary=[], modular_debug=[], modular_source=[] + mock_current_content, modular_binary=[], modular_debug=[], modular_source=[] ) _, _, srpms, _ = associations @@ -1124,9 +1082,16 @@ def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft): def test_associate_units(mock_ubipop_runner): src_repo = get_test_repo(id="test_src") dst_repo = get_test_repo(id="test_dst") - + unit = get_modulemd_unit( + name="test_assoc", + stream="fake-stream", + version=1, + context="fake-context", + arch="x86_64", + src_repo_id=src_repo.id, + ) associations = [ - AssociateActionModules([get_test_mod(name="test_assoc")], dst_repo, [src_repo]), + AssociateActionModules([unit], dst_repo, [src_repo]), ] mock_ubipop_runner.pulp.associate_modules.return_value = ["task_id"] @@ -1141,27 +1106,30 @@ def test_associate_units(mock_ubipop_runner): def test_associate_unassociate_md_defaults(mock_ubipop_runner): src_repo = get_test_repo(id="test_src") dst_repo = get_test_repo(id="tets_dst") + unit_1 = get_modulemd_defaults_unit( + name="virt", + stream="rhel", + profiles={"2.5": ["common"]}, + repo_id="test_src", + src_repo_id="test_src", + ) + + unit_2 = get_modulemd_defaults_unit( + name="virt", + stream="rhel", + profiles={"2.5": ["unique"]}, + repo_id="test_src", + src_repo_id="test_src", + ) associations = AssociateActionModuleDefaults( - [ - get_test_mod_defaults( - name="virt", - stream="rhel", - profiles={"2.5": ["common"]}, - ), - ], + [unit_1], dst_repo, [src_repo], ) unassociations = UnassociateActionModuleDefaults( - [ - get_test_mod_defaults( - name="virt", - stream="rhel", - profiles={"2.5": ["unique"]}, - ), - ], + [unit_2], dst_repo, ) @@ -1177,3 +1145,91 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): # the calls has to be in order calls = [call(["task_id_0"]), call(["task_id_1"])] mock_ubipop_runner.pulp.wait_for_tasks.assert_has_calls(calls) + + +@pytest.mark.parametrize( + "skip_debug_repo", + [True, False], + ids=["skip_debug_repo_true", "skip_debug_repo_false"], +) +def test_get_current_content(mock_ubipop_runner, pulp, skip_debug_repo): + """Tests getting current content from ubi repos, using Fake Client from pubtools-pulplib""" + rpm_repo = YumRepository( + id="rpm_repo", + ) + rpm_repo.__dict__["_client"] = pulp.client + + debug_repo = YumRepository( + id="debug_repo", + ) + debug_repo.__dict__["_client"] = pulp.client + + source_repo = YumRepository( + id="source_repo", + ) + source_repo.__dict__["_client"] = pulp.client + + binary_rpm = RpmUnit(name="test", version="1.0", release="1", arch="x86_64") + modulemd = ModulemdUnit( + name="test_module_md", + stream="fake-stream", + version=1, + context="fake-context", + arch="x86_64", + ) + + modulemd_defaults = ModulemdDefaultsUnit( + name="test_modulemd_defaults", stream="rhel", repo_id="rpm_repo" + ) + + debug_rpm = RpmUnit( + name="test-debuginfo", version="1.0", release="1", arch="x86_64" + ) + source_rpm = RpmUnit( + name="test-srpm", version="1.0", release="1", arch="src", content_type_id="srpm" + ) + + pulp.insert_repository(rpm_repo) + pulp.insert_units(rpm_repo, [binary_rpm, modulemd, modulemd_defaults]) + + if not skip_debug_repo: + pulp.insert_repository(debug_repo) + pulp.insert_units(debug_repo, [debug_rpm]) + + pulp.insert_repository(source_repo) + pulp.insert_units(source_repo, [source_rpm]) + + if skip_debug_repo: + debug_repo = f_return(None) + else: + debug_repo = f_proxy(f_return(debug_repo)) + + # overwrite out_repos with the testing ones + mock_ubipop_runner.repos.out_repos = RepoSet( + f_proxy(f_return(rpm_repo)), f_proxy(f_return(source_repo)), debug_repo + ) + + content = mock_ubipop_runner._get_current_content() + + binary_rpms = list(content.binary_rpms) + assert len(binary_rpms) == 1 + assert binary_rpms[0].name == "test" + + modules = list(content.modules) + assert len(modules) == 1 + assert modules[0].name == "test_module_md" + + modulemd_defaults = list(content.modulemd_defaults) + assert len(modulemd_defaults) == 1 + assert modulemd_defaults[0].name == "test_modulemd_defaults" + + debug_rpms = list(content.debug_rpms) + if skip_debug_repo: + assert len(debug_rpms) == 0 + else: + assert len(debug_rpms) == 1 + assert debug_rpms[0].name == "test-debuginfo" + + source_rpms = list(content.source_rpms) + assert len(source_rpms) == 1 + assert source_rpms[0].name == "test-srpm" diff --git a/tests/test_utils.py b/tests/test_utils.py index 8f788bc..840d98b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from mock import MagicMock -from pubtools.pulplib import YumRepository, RpmUnit +from pubtools.pulplib import YumRepository, RpmUnit, ModulemdDefaultsUnit from ubipop._utils import ( AssociateAction, AssociateActionModuleDefaults, @@ -12,6 +12,7 @@ UnassociateActionModules, UnassociateActionRpms, vercmp_sort, + flatten_md_defaults_name_profiles, ) from ubipop._matcher import UbiUnit @@ -121,3 +122,19 @@ def test_vercmp_sort(): assert (unit_1 >= unit_2) is False assert (unit_1 > unit_2) is False assert (unit_1 != unit_2) is True + + +def test_flatten_md_defaults_name_profiles(): + unit = UbiUnit( + ModulemdDefaultsUnit( + name="test", + stream="foo", + profiles={"rhel": ["common", "uncommon"], "fedora": ["super", "ultra"]}, + repo_id="foo-repo", + ), + "foo-repo", + ) + + out = flatten_md_defaults_name_profiles(unit) + + assert out == "test:[fedora:super,ultra]:[rhel:common,uncommon]" diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 8079479..518c20b 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -7,21 +7,22 @@ from itertools import chain from pubtools.pulplib import Client, Criteria, PublishOptions +import attr import ubiconfig from more_executors import Executors from more_executors.futures import f_sequence, f_proxy, f_return -from ubipop._pulp_client import Pulp, Package +from ubipop._pulp_client import Pulp from ubipop._utils import ( - split_filename, AssociateActionModules, AssociateActionModuleDefaults, AssociateActionRpms, UnassociateActionModules, UnassociateActionModuleDefaults, UnassociateActionRpms, + flatten_md_defaults_name_profiles, ) -from ._matcher import ModularMatcher, RpmMatcher +from ._matcher import ModularMatcher, RpmMatcher, Matcher _LOG = logging.getLogger("ubipop") @@ -42,6 +43,15 @@ class PopulationSourceMissing(Exception): RepoSet = namedtuple("RepoSet", ["rpm", "source", "debug"]) +@attr.s +class RepoContent(object): + binary_rpms = attr.ib() + source_rpms = attr.ib() + debug_rpms = attr.ib() + modules = attr.ib() + modulemd_defaults = attr.ib() + + class UbiRepoSet(object): def __init__(self, input_repos, output_repos): self.in_repos = input_repos @@ -378,27 +388,6 @@ def __init__( self.dry_run = dry_run self._executor = executor - def _match_module_defaults(self): - """Try to find modulemd_defaults units in the same repo with the same - name/stream of a modulemd. - """ - fts = {} - for module in self.repos.modules: - for in_repo_rpm in self.repos.in_repos.rpm: - fts[ - self._executor.submit( - self.pulp.search_module_defaults, - in_repo_rpm, - module.name, - str(module.stream), - ) - ] = module.name + str(module.stream) - - for ft in as_completed(fts): - module_defaults = ft.result() - if module_defaults: - self.repos.module_defaults[fts[ft]].extend(module_defaults) - def _determine_pulp_actions(self, units, current, diff_f, extra_units=None): expected = list(units) if extra_units: @@ -414,9 +403,8 @@ def _get_pulp_actions_mds(self, modules, current): ) def _get_pulp_actions_md_defaults(self, module_defaults, current): - module_defaults_list = list(chain.from_iterable(module_defaults.values())) return self._determine_pulp_actions( - module_defaults_list, + module_defaults, current, self._diff_md_defaults_by_profiles, ) @@ -447,11 +435,7 @@ def _get_pulp_actions_src_pkgs(self, pkgs, current, modular): def _get_pulp_actions( self, - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, + current_content, modular_binary, modular_debug, modular_source, @@ -466,24 +450,24 @@ def _get_pulp_actions( """ modules_assoc, modules_unassoc = self._get_pulp_actions_mds( - self.repos.modules, current_modules_ft.result() + self.repos.modules, current_content.modules ) md_defaults_assoc, md_defaults_unassoc = self._get_pulp_actions_md_defaults( - self.repos.module_defaults, current_module_defaults_ft.result() + self.repos.module_defaults, current_content.modulemd_defaults ) rpms_assoc, rpms_unassoc = self._get_pulp_actions_pkgs( - self.repos.packages, current_rpms_ft.result(), modular_binary + self.repos.packages, current_content.binary_rpms, modular_binary ) srpms_assoc, srpms_unassoc = self._get_pulp_actions_src_pkgs( - self.repos.source_rpms, current_srpms_ft.result(), modular_source + self.repos.source_rpms, current_content.source_rpms, modular_source ) debug_assoc = None debug_unassoc = None - if current_debug_rpms_ft is not None: + if current_content.debug_rpms: debug_assoc, debug_unassoc = self._get_pulp_actions_pkgs( - self.repos.debug_rpms, current_debug_rpms_ft.result(), modular_debug + self.repos.debug_rpms, current_content.debug_rpms, modular_debug ) associations = ( @@ -523,62 +507,49 @@ def _diff_modules_by_nsvca(self, modules_1, modules_2): def _diff_md_defaults_by_profiles(self, module_defaults_1, module_defaults_2): return self._diff_lists_by_attr( - module_defaults_1, module_defaults_2, "name_profiles" + module_defaults_1, module_defaults_2, flatten_md_defaults_name_profiles ) def _diff_packages_by_filename(self, packages_1, packages_2): return self._diff_lists_by_attr(packages_1, packages_2, "filename") - def _diff_lists_by_attr(self, list_1, list_2, attr): - attrs_list_2 = [getattr(obj, attr) for obj in list_2] - diff = [obj for obj in list_1 if getattr(obj, attr) not in attrs_list_2] + def _diff_lists_by_attr(self, list_1, list_2, attr_or_func): + def diff_attr(obj): + if callable(attr_or_func): + return attr_or_func(obj) + return getattr(obj, attr_or_func) + + attrs_list_2 = [diff_attr(obj) for obj in list_2] + diff = [obj for obj in list_1 if diff_attr(obj) not in attrs_list_2] return diff def run_ubi_population(self): - ( - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, - ) = self._get_current_content() - + current_content = self._get_current_content() # start async querying for modulemds and modular and non-modular packages mm = ModularMatcher(self.repos.in_repos, self.ubiconfig.modules).run() rm = RpmMatcher(self.repos.in_repos, self.ubiconfig).run() self.repos.modules = mm.modules + self.repos.module_defaults = mm.modulemd_defaults self.repos.packages = rm.binary_rpms self.repos.debug_rpms = rm.debug_rpms self.repos.source_rpms = rm.source_rpms - self._match_module_defaults() - ( associations, unassociations, mdd_association, mdd_unassociation, ) = self._get_pulp_actions( - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, + current_content, mm.binary_rpms, mm.debug_rpms, mm.source_rpms, ) if self.dry_run: - self.log_curent_content( - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, - ) + self.log_curent_content(current_content) self.log_pulp_actions( associations + (mdd_association,), unassociations + (mdd_unassociation,), @@ -624,33 +595,26 @@ def _wait_pulp(self, futures): if tasks: self.pulp.wait_for_tasks(tasks) - def log_curent_content( - self, - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, - ): + def log_curent_content(self, current_content): _LOG.info("Current modules in repo: %s", self.repos.out_repos.rpm.id) - for module in current_modules_ft.result(): + for module in current_content.modules: _LOG.info(module.nsvca) _LOG.info("Current module_defaults in repo: %s", self.repos.out_repos.rpm.id) - for md_d in current_module_defaults_ft.result(): + for md_d in current_content.modulemd_defaults: _LOG.info("module_defaults: %s, profiles: %s", md_d.name, md_d.profiles) _LOG.info("Current rpms in repo: %s", self.repos.out_repos.rpm.id) - for rpm in current_rpms_ft.result(): + for rpm in current_content.binary_rpms: _LOG.info(rpm.filename) _LOG.info("Current srpms in repo: %s", self.repos.out_repos.source.id) - for rpm in current_srpms_ft.result(): + for rpm in current_content.source_rpms: _LOG.info(rpm.filename) if self.repos.out_repos.debug: _LOG.info("Current rpms in repo: %s", self.repos.out_repos.debug.id) - for rpm in current_debug_rpms_ft.result(): + for rpm in current_content.debug_rpms: _LOG.info(rpm.filename) def log_pulp_actions(self, associations, unassociations): @@ -687,32 +651,45 @@ def _get_current_content(self): """ Gather current content of output repos """ - current_modules_ft = self._executor.submit( - self.pulp.search_modules, self.repos.out_repos.rpm + criteria = [Criteria.true()] + current_modulemds = f_proxy( + self._executor.submit( + Matcher.search_modulemds, criteria, [self.repos.out_repos.rpm] + ) ) - current_module_defaults_ft = self._executor.submit( - self.pulp.search_module_defaults, self.repos.out_repos.rpm + current_modulemd_defaults = f_proxy( + self._executor.submit( + Matcher.search_modulemd_defaults, criteria, [self.repos.out_repos.rpm] + ) ) - current_rpms_ft = self._executor.submit( - self.pulp.search_rpms, self.repos.out_repos.rpm + current_rpms = f_proxy( + self._executor.submit( + Matcher.search_rpms, criteria, [self.repos.out_repos.rpm] + ) ) - current_srpms_ft = self._executor.submit( - self.pulp.search_rpms, self.repos.out_repos.source + current_srpms = f_proxy( + self._executor.submit( + Matcher.search_srpms, criteria, [self.repos.out_repos.source] + ) ) - if self.repos.out_repos.debug: - current_debug_rpms_ft = self._executor.submit( - self.pulp.search_rpms, self.repos.out_repos.debug + + if self.repos.out_repos.debug.result(): + current_debug_rpms = f_proxy( + self._executor.submit( + Matcher.search_rpms, criteria, [self.repos.out_repos.debug] + ) ) else: - current_debug_rpms_ft = None - - return ( - current_modules_ft, - current_module_defaults_ft, - current_rpms_ft, - current_srpms_ft, - current_debug_rpms_ft, + current_debug_rpms = f_proxy(f_return([])) + + current_content = RepoContent( + current_rpms, + current_srpms, + current_debug_rpms, + current_modulemds, + current_modulemd_defaults, ) + return current_content def _publish_out_repos(self): fts = [] diff --git a/ubipop/_matcher.py b/ubipop/_matcher.py index 696b15f..d545025 100644 --- a/ubipop/_matcher.py +++ b/ubipop/_matcher.py @@ -66,8 +66,9 @@ def run(self): """ raise NotImplementedError + @classmethod def _search_units( - self, repo, criteria_list, content_type_id, batch_size_override=None + cls, repo, criteria_list, content_type_id, batch_size_override=None ): """ Search for units of one content type associated with given repository by criteria. @@ -122,13 +123,14 @@ def _create_or_criteria(self, fields, values): return or_criteria + @classmethod def _search_units_per_repos( - self, or_criteria, repos, content_type, batch_size_override=None + cls, or_criteria, repos, content_type, batch_size_override=None ): units = [] for repo in repos: units.append( - self._search_units( + cls._search_units( repo, or_criteria, content_type, @@ -138,24 +140,33 @@ def _search_units_per_repos( return f_proxy(f_flat_map(f_sequence(units), flatten_list_of_sets)) - def _search_rpms(self, or_criteria, repos, batch_size_override=None): - return self._search_units_per_repos( + @classmethod + def search_rpms(cls, or_criteria, repos, batch_size_override=None): + return cls._search_units_per_repos( or_criteria, repos, content_type="rpm", batch_size_override=batch_size_override, ) - def _search_srpms(self, or_criteria, repos, batch_size_override=None): - return self._search_units_per_repos( + @classmethod + def search_srpms(cls, or_criteria, repos, batch_size_override=None): + return cls._search_units_per_repos( or_criteria, repos, content_type="srpm", batch_size_override=batch_size_override, ) - def _search_moludemds(self, or_criteria, repos): - return self._search_units_per_repos(or_criteria, repos, content_type="modulemd") + @classmethod + def search_modulemds(cls, or_criteria, repos): + return cls._search_units_per_repos(or_criteria, repos, content_type="modulemd") + + @classmethod + def search_modulemd_defaults(cls, or_criteria, repos): + return cls._search_units_per_repos( + or_criteria, repos, content_type="modulemd_defaults" + ) def _get_srpms_criteria(self): filenames = [] @@ -176,6 +187,7 @@ class ModularMatcher(Matcher): def __init__(self, input_repos, ubi_config): super(ModularMatcher, self).__init__(input_repos, ubi_config) self.modules = None + self.modulemd_defaults = None def run(self): """Asynchronously creates criteria for pulp queries and @@ -189,7 +201,7 @@ def run(self): ) modules = f_proxy( self._executor.submit( - self._search_moludemds, modulemds_criteria, self._input_repos.rpm + self.search_modulemds, modulemds_criteria, self._input_repos.rpm ) ) self.modules = f_proxy( @@ -198,18 +210,28 @@ def run(self): rpms_criteria = f_proxy(self._executor.submit(self._get_modular_rpms_criteria)) self.binary_rpms = f_proxy( self._executor.submit( - self._search_rpms, rpms_criteria, self._input_repos.rpm + self.search_rpms, rpms_criteria, self._input_repos.rpm ) ) self.debug_rpms = f_proxy( self._executor.submit( - self._search_rpms, rpms_criteria, self._input_repos.debug + self.search_rpms, rpms_criteria, self._input_repos.debug ) ) srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) self.source_rpms = f_proxy( self._executor.submit( - self._search_srpms, srpms_criteria, self._input_repos.source + self.search_srpms, srpms_criteria, self._input_repos.source + ) + ) + modulemd_defaults_criteria = f_proxy( + self._executor.submit(self._get_modulemd_defaults_criteria) + ) + self.modulemd_defaults = f_proxy( + self._executor.submit( + self.search_modulemd_defaults, + modulemd_defaults_criteria, + self._input_repos.rpm, ) ) return self @@ -221,8 +243,14 @@ def _get_modular_rpms_criteria(self): return pkgs_or_criteria def _get_modulemds_criteria(self): + return self._get_criteria_for_modules(self._ubi_config) + + def _get_modulemd_defaults_criteria(self): + return self._get_criteria_for_modules(self.modules) + + def _get_criteria_for_modules(self, modules): criteria_values = [] - for module in self._ubi_config: + for module in modules: criteria_values.append( ( module.name, @@ -317,7 +345,7 @@ def run(self): binary_rpms = f_proxy( self._executor.submit( - self._search_rpms, + self.search_rpms, rpms_criteria, self._input_repos.rpm, batch_size_override, @@ -326,7 +354,7 @@ def run(self): debug_rpms = f_proxy( self._executor.submit( - self._search_rpms, + self.search_rpms, rpms_criteria, self._input_repos.debug, batch_size_override, @@ -347,7 +375,7 @@ def run(self): srpms_criteria = f_proxy(self._executor.submit(self._get_srpms_criteria)) source_rpms = f_proxy( self._executor.submit( - self._search_srpms, + self.search_srpms, srpms_criteria, self._input_repos.source, batch_size_override, @@ -396,7 +424,7 @@ def extract_modular_filenames(): return modular_rpm_filenames - modules = self._search_moludemds([Criteria.true()], self._input_repos.rpm) + modules = self.search_modulemds([Criteria.true()], self._input_repos.rpm) return self._executor.submit(extract_modular_filenames) def _get_rpm_output_set( diff --git a/ubipop/_pulp_client.py b/ubipop/_pulp_client.py index b27ecce..9dc93d4 100644 --- a/ubipop/_pulp_client.py +++ b/ubipop/_pulp_client.py @@ -4,7 +4,6 @@ import time from urllib3.util.retry import Retry -from ubipop._utils import split_filename try: from urllib.parse import urljoin @@ -90,91 +89,6 @@ def do_request(self, req_type, url, data=None): return ret - def search_rpms( - self, repo, name=None, arch=None, name_globbing=False, filename=None - ): - url = "repositories/{REPO_ID}/search/units/".format(REPO_ID=repo.id) - criteria = {"type_ids": ["rpm", "srpm"]} - - filters = {"filters": {"unit": {}}} - if name: - if name_globbing: - filters["filters"]["unit"]["name"] = {"$regex": name + ".*"} - else: - filters["filters"]["unit"]["name"] = name - - if arch: - filters["filters"]["unit"]["arch"] = arch - - if filename: - filters["filters"]["unit"]["filename"] = filename - - criteria.update(filters) - payload = {"criteria": criteria} - ret = self.do_request("post", url, payload) - rpms = [] - ret.raise_for_status() - for item in ret.json(): - metadata = item["metadata"] - rpms.append( - Package( - metadata["name"], - metadata["filename"], - repo.id, - sourcerpm_filename=metadata.get("sourcerpm"), - ) - ) - return rpms - - def search_modules(self, repo, name=None, stream=None): - url = "repositories/{REPO_ID}/search/units/".format(REPO_ID=repo.id) - criteria = {"type_ids": ["modulemd"]} - if name and stream: - criteria.update({"filters": {"unit": {"name": name, "stream": stream}}}) - payload = {"criteria": criteria} - - ret = self.do_request("post", url, payload) - modules = [] - ret.raise_for_status() - for item in ret.json(): - - metadata = item["metadata"] - modules.append( - Module( - metadata["name"], - metadata["stream"], - metadata["version"], - metadata["context"], - metadata["arch"], - metadata["artifacts"], - metadata["profiles"], - repo.id, - ) - ) - return modules - - def search_module_defaults(self, repo, name=None, stream=None): - url = "repositories/{REPO_ID}/search/units/".format(REPO_ID=repo.id) - criteria = {"type_ids": ["modulemd_defaults"]} - if name and stream: - criteria.update({"filters": {"unit": {"name": name, "stream": stream}}}) - payload = {"criteria": criteria} - - ret = self.do_request("post", url, payload) - ret.raise_for_status() - module_defaults = [] - for item in ret.json(): - metadata = item["metadata"] - module_defaults.append( - ModuleDefaults( - metadata["name"], - metadata["stream"], - metadata["profiles"], - repo.id, - ) - ) - return module_defaults - def wait_for_tasks(self, task_id_list, delay=5.0): results = {} @@ -290,80 +204,3 @@ def unassociate_module_defaults(self, repo, module_defaults): def unassociate_packages(self, repo, rpms): return self.unassociate_units(repo, rpms, ("rpm", "srpm")) - - -class Package(object): - def __init__( - self, name, filename, src_repo_id, sourcerpm_filename=None, is_modular=False - ): - self.name = name - self.filename = filename - self.sourcerpm = sourcerpm_filename - self.is_modular = is_modular - # return name, ver, rel, epoch, arch - _, self.version, self.release, self.epoch, _ = split_filename(self.filename) - self.associate_source_repo_id = src_repo_id - - def __str__(self): - return self.filename - - -class Module(object): - def __init__( - self, name, stream, version, context, arch, packages, profiles, src_repo_id - ): - self.name = name - self.stream = stream - self.version = version - self.context = context - self.arch = arch - self.packages = packages - self.profiles = profiles - self.associate_source_repo_id = src_repo_id - - @property - def nsvca(self): - return ":".join( - (self.name, self.stream, str(self.version), self.context, self.arch) - ) - - def __str__(self): - return self.nsvca - - -class ModuleDefaults(object): - """ - module_defaults unit, defines which profiles are enabled by default when activating - a module. For example: - {..., - "name": "ruby", - "profiles": { - "2.5": [ - "common"] - }, - ... - } - if someone asks to enable 'ruby:2.5' for some repo without specifing profiles, will - get 'common' profile by defualt - """ - - def __init__(self, name, stream, profiles, src_repo_id): - self.name = name - self.stream = stream - self.profiles = profiles # a dict such as {'4.046':['common']} - self.associate_source_repo_id = src_repo_id - - def __str__(self): - return self.name - - @property - def name_profiles(self): - """ - flatten the profles and prepend name - format: name:[key:profile,profile]:[key:profile] - 'ruby:[2.5:common,unique]' - """ - result = self.name - for key in sorted(self.profiles): - result += ":[%s:%s]" % (key, ",".join(sorted(self.profiles[key]))) - return result diff --git a/ubipop/_utils.py b/ubipop/_utils.py index 07f609f..06f259b 100644 --- a/ubipop/_utils.py +++ b/ubipop/_utils.py @@ -165,3 +165,15 @@ def __ne__(self, other): return label_compare(self.evr_tuple, other.evr_tuple) != 0 return Klass + + +def flatten_md_defaults_name_profiles(obj): + """ + flatten the profiles of md_defaults unit and prepend name + format: name:[key:profile,profile]:[key:profile] + 'ruby:[2.5:common,unique]' + """ + result = obj.name + for key in sorted(obj.profiles): + result += ":[%s:%s]" % (key, ",".join(sorted(obj.profiles[key]))) + return result diff --git a/ubipop/test_attr.py b/ubipop/test_attr.py new file mode 100644 index 0000000..784251c --- /dev/null +++ b/ubipop/test_attr.py @@ -0,0 +1,12 @@ +import attr +import inspect + + +@attr.s +class Coordinates(object): + x = attr.ib() + + +print(inspect.getsource(Coordinates.__init__)) +##xxx = XXX(100, 200)# +# print(xxx.x)