Merge f71ad45 into 7878eda

tests/test_matcher.py

@@ -8,6 +8,7 @@
     YumRepository,
     FakeController,
     ModulemdUnit,
+    ModulemdDefaultsUnit,
 )
 from ubiconfig import UbiConfig
 
@@ -280,6 +281,37 @@ def test_search_moludemds(pulp):
     assert result.pop().nsvca == "test:10:100:abcdef:x86_64"
 
 
+def test_search_moludemd_defaults(pulp):
+    """Test convenient method for searching modulemd_defaults"""
+    repo = YumRepository(
+        id="test_repo_1",
+    )
+    repo.__dict__["_client"] = pulp.client
+    unit_1 = ModulemdDefaultsUnit(
+        name="test",
+        stream="10",
+        repo_id="test_repo_1",
+    )
+    unit_2 = ModulemdDefaultsUnit(
+        name="test",
+        stream="20",
+        repo_id="test_repo_1",
+    )
+
+    pulp.insert_repository(repo)
+    pulp.insert_units(repo, [unit_1, unit_2])
+
+    matcher = Matcher(None, None)
+    criteria = matcher._create_or_criteria(["name", "stream"], [("test", "10")])
+    # let Future return result
+    result = matcher._search_modulemd_defaults(criteria, [repo]).result()
+    # there should be be only one unit in the result set according to criteria
+    assert len(result) == 1
+    found_unit = result.pop()
+    assert found_unit.name == "test"
+    assert found_unit.stream == "10"
+
+
 def test_modular_rpms_filenames(ubi_config):
     """Test getting filename from module artifacts, srpms are skipped."""
     matcher = ModularMatcher(None, ubi_config.modules)
@@ -490,6 +522,25 @@ def test_get_modulemds_criteria(ubi_config):
     # let's not test internal structure of criteria, that's responsibility of pulplib
 
 
+def test_get_modulemd_defaults_criteria():
+    """Test proper creation of criteria for modulemd_defaults query"""
+    matcher = ModularMatcher(None, None)
+    unit = UbiUnit(
+        ModulemdUnit(
+            name="test", stream="10", version=100, context="abcd", arch="x86_64"
+        ),
+        None,
+    )
+    matcher.modules = [unit]
+    criteria = matcher._get_modulemd_defaults_criteria()
+    # there should be 1 criterium created based on modules list of Matcher obj.
+    assert len(criteria) == 1
+    # it should be instance of Criteria
+    for crit in criteria:
+        assert isinstance(crit, Criteria)
+    # let's not test internal structure of criteria, that's responsibility of pulplib
+
+
 def test_get_modular_srpms_criteria(ubi_config):
     """Testing creation of criteria for srpms query"""
     matcher = ModularMatcher(None, ubi_config.modules)
@@ -614,10 +665,16 @@ def test_modular_matcher_run(pulp, ubi_config):
             "test-7:1.0-1.x86_64.src",
         ],
     )
+
+    modulemd_defaults = ModulemdDefaultsUnit(
+        name="fake_name",
+        stream="fake_stream",
+        repo_id="binary_repo",
+    )
     pulp.insert_repository(repo_1)
     pulp.insert_repository(repo_2)
     pulp.insert_repository(repo_3)
-    pulp.insert_units(repo_1, [unit_1, modulemd])
+    pulp.insert_units(repo_1, [unit_1, modulemd, modulemd_defaults])
     pulp.insert_units(repo_2, [unit_2])
     pulp.insert_units(repo_3, [unit_3])
 
@@ -627,6 +684,7 @@ def test_modular_matcher_run(pulp, ubi_config):
 
     # each public attribute is properly set with one unit
     assert len(matcher.modules) == 1
+    assert len(matcher.modulemd_defaults) == 1
     assert len(matcher.binary_rpms) == 1
     assert len(matcher.debug_rpms) == 1
     assert len(matcher.source_rpms) == 1
@@ -636,6 +694,10 @@ def test_modular_matcher_run(pulp, ubi_config):
     assert output_module.nsvca == "fake_name:fake_stream:100:abcd:x86_64"
     assert output_module.associate_source_repo_id == "binary_repo"
 
+    output_modulemd_defaults = matcher.modulemd_defaults.pop()
+    assert output_modulemd_defaults.name == "fake_name"
+    assert output_modulemd_defaults.stream == "fake_stream"
+
     rpm = matcher.binary_rpms.pop()
     assert rpm.filename == "test-1.0-1.x86_64.x86_64.rpm"
     assert rpm.associate_source_repo_id == "binary_repo"

tests/test_pulp.py

@@ -229,7 +229,6 @@ def test_search_module_defaults(mock_pulp, mock_search_module_defaults, mock_rep
     assert len(found_module_defaults) == 1
     assert found_module_defaults[0].name == "virt"
     assert found_module_defaults[0].stream == "rhel"
-    assert found_module_defaults[0].name_profiles == "virt:[rhel:common,default]"
 
 
 @pytest.fixture(name="search_task_response")

tests/test_ubipop.py

@@ -16,6 +16,7 @@
     Distributor,
     ModulemdUnit,
     RpmUnit,
+    ModulemdDefaultsUnit,
 )
 from mock import MagicMock, patch, call
 from more_executors import Executors
@@ -385,32 +386,6 @@ def _f(*args, **kwargs):
     return _f
 
 
-def test_match_module_defaults(mock_ubipop_runner):
-    unit = UbiUnit(
-        ModulemdUnit(
-            name="virt",
-            stream="rhel",
-            version=1,
-            context="abcd",
-            arch="x86_64",
-            profiles={"2.5": ["common"]},
-        ),
-        None,
-    )
-    mock_ubipop_runner.repos.modules = f_proxy(f_return(set([unit])))
-    mock_ubipop_runner.pulp.search_module_defaults.return_value = [
-        get_test_mod_defaults(name="virt", stream="rhel", profiles={"2.5": ["common"]}),
-    ]
-
-    mock_ubipop_runner._match_module_defaults()  # pylint: disable=protected-access
-
-    assert len(mock_ubipop_runner.repos.module_defaults) == 1
-    md_d = mock_ubipop_runner.repos.module_defaults["virtrhel"]
-    assert len(md_d) == 1
-    assert md_d[0].name == "virt"
-    assert md_d[0].name_profiles == "virt:[2.5:common]"
-
-
 def test_diff_modules(mock_ubipop_runner):
     curr = [
         get_test_mod(name="1"),
@@ -714,13 +689,17 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft):
         f_return(set([get_test_mod(name="test_md", pulplib=True)]))
     )
 
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="test_mdd", stream="rhel", profiles={"2.5": "uncommon"}
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="test_mdd",
+                stream="rhel",
+                profiles={"2.5": "uncommon"},
+                repo_id="foo-rpms",
             ),
-        ],
-    }
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(
@@ -872,13 +851,18 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft
     mock_ubipop_runner.repos.modules = f_proxy(
         f_return(set([get_test_mod(name="md_current", pulplib=True)]))
     )
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="mdd_current", stream="rhel", profiles={"2.5": "common"}
+
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="mdd_current",
+                stream="rhel",
+                profiles={"2.5": "common"},
+                repo_id="foo-rpms",
             ),
-        ],
-    }
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(
@@ -1012,13 +996,18 @@ def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft):
     mock_ubipop_runner.repos.modules = f_proxy(
         f_return(set([get_test_mod(name="md_current", pulplib=True)]))
     )
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="mdd_current", stream="rhel", profiles={"2.5": "common"}
-            )
-        ]
-    }
+
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="mdd_current",
+                stream="rhel",
+                profiles={"2.5": "common"},
+                repo_id="foo-rpms",
+            ),
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(

tests/test_utils.py

@@ -1,7 +1,7 @@
 import pytest
 
 from mock import MagicMock
-from pubtools.pulplib import YumRepository, RpmUnit
+from pubtools.pulplib import YumRepository, RpmUnit, ModulemdDefaultsUnit
 from ubipop._utils import (
     AssociateAction,
     AssociateActionModuleDefaults,
@@ -12,6 +12,7 @@
     UnassociateActionModules,
     UnassociateActionRpms,
     vercmp_sort,
+    flatten_md_defaults_name_profiles,
 )
 from ubipop._matcher import UbiUnit
 
@@ -121,3 +122,19 @@ def test_vercmp_sort():
     assert (unit_1 >= unit_2) is False
     assert (unit_1 > unit_2) is False
     assert (unit_1 != unit_2) is True
+
+
+def test_flatten_md_defaults_name_profiles():
+    unit = UbiUnit(
+        ModulemdDefaultsUnit(
+            name="test",
+            stream="foo",
+            profiles={"rhel": ["common", "uncommon"], "fedora": ["super", "ultra"]},
+            repo_id="foo-repo",
+        ),
+        "foo-repo",
+    )
+
+    out = flatten_md_defaults_name_profiles(unit)
+
+    assert out == "test:[fedora:super,ultra]:[rhel:common,uncommon]"

ubipop/__init__.py

@@ -20,6 +20,7 @@
     UnassociateActionModules,
     UnassociateActionModuleDefaults,
     UnassociateActionRpms,
+    flatten_md_defaults_name_profiles,
 )
 from ._matcher import ModularMatcher, RpmMatcher
 
@@ -378,27 +379,6 @@ def __init__(
         self.dry_run = dry_run
         self._executor = executor
 
-    def _match_module_defaults(self):
-        """Try to find modulemd_defaults units in the same repo with the same
-        name/stream of a modulemd.
-        """
-        fts = {}
-        for module in self.repos.modules:
-            for in_repo_rpm in self.repos.in_repos.rpm:
-                fts[
-                    self._executor.submit(
-                        self.pulp.search_module_defaults,
-                        in_repo_rpm,
-                        module.name,
-                        str(module.stream),
-                    )
-                ] = module.name + str(module.stream)
-
-        for ft in as_completed(fts):
-            module_defaults = ft.result()
-            if module_defaults:
-                self.repos.module_defaults[fts[ft]].extend(module_defaults)
-
     def _determine_pulp_actions(self, units, current, diff_f, extra_units=None):
         expected = list(units)
         if extra_units:
@@ -414,9 +394,8 @@ def _get_pulp_actions_mds(self, modules, current):
         )
 
     def _get_pulp_actions_md_defaults(self, module_defaults, current):
-        module_defaults_list = list(chain.from_iterable(module_defaults.values()))
         return self._determine_pulp_actions(
-            module_defaults_list,
+            module_defaults,
             current,
             self._diff_md_defaults_by_profiles,
         )
@@ -523,15 +502,20 @@ def _diff_modules_by_nsvca(self, modules_1, modules_2):
 
     def _diff_md_defaults_by_profiles(self, module_defaults_1, module_defaults_2):
         return self._diff_lists_by_attr(
-            module_defaults_1, module_defaults_2, "name_profiles"
+            module_defaults_1, module_defaults_2, flatten_md_defaults_name_profiles
         )
 
     def _diff_packages_by_filename(self, packages_1, packages_2):
         return self._diff_lists_by_attr(packages_1, packages_2, "filename")
 
-    def _diff_lists_by_attr(self, list_1, list_2, attr):
-        attrs_list_2 = [getattr(obj, attr) for obj in list_2]
-        diff = [obj for obj in list_1 if getattr(obj, attr) not in attrs_list_2]
+    def _diff_lists_by_attr(self, list_1, list_2, attr_or_func):
+        def diff_attr(obj):
+            if callable(attr_or_func):
+                return attr_or_func(obj)
+            return getattr(obj, attr_or_func)
+
+        attrs_list_2 = [diff_attr(obj) for obj in list_2]
+        diff = [obj for obj in list_1 if diff_attr(obj) not in attrs_list_2]
 
         return diff
 
@@ -549,12 +533,11 @@ def run_ubi_population(self):
         rm = RpmMatcher(self.repos.in_repos, self.ubiconfig).run()
 
         self.repos.modules = mm.modules
+        self.repos.module_defaults = mm.modulemd_defaults
         self.repos.packages = rm.binary_rpms
         self.repos.debug_rpms = rm.debug_rpms
         self.repos.source_rpms = rm.source_rpms
 
-        self._match_module_defaults()
-
         (
             associations,
             unassociations,