Merge 8eea0aa into 65d12ec

tests/test_matcher.py

@@ -8,6 +8,7 @@
     YumRepository,
     FakeController,
     ModulemdUnit,
+    ModulemdDefaultsUnit,
 )
 from ubiconfig import UbiConfig
 
@@ -280,6 +281,39 @@ def test_search_moludemds(pulp):
     assert result.pop().nsvca == "test:10:100:abcdef:x86_64"
 
 
+def test_search_moludemd_defaults(pulp):
+    """Test convenient method for searching modulemd_defaults"""
+    repo = YumRepository(
+        id="test_repo_1",
+    )
+    repo.__dict__["_client"] = pulp.client
+    unit_1 = ModulemdDefaultsUnit(
+        name="test",
+        stream="10",
+        repo_id="test_repo_1",
+        content_type_id="modulemd_defaults",
+    )
+    unit_2 = ModulemdDefaultsUnit(
+        name="test",
+        stream="20",
+        repo_id="test_repo_1",
+        content_type_id="modulemd_defaults",
+    )
+
+    pulp.insert_repository(repo)
+    pulp.insert_units(repo, [unit_1, unit_2])
+
+    matcher = Matcher(None, None)
+    criteria = matcher._create_or_criteria(["name", "stream"], [("test", "10")])
+    # let Future return result
+    result = matcher._search_modulemd_defaults(criteria, [repo]).result()
+    # there should be be only one unit in the result set according to criteria
+    assert len(result) == 1
+    found_unit = result.pop()
+    assert found_unit.name == "test"
+    assert found_unit.stream == "10"
+
+
 def test_modular_rpms_filenames(ubi_config):
     """Test getting filename from module artifacts, srpms are skipped."""
     matcher = ModularMatcher(None, ubi_config.modules)
@@ -490,6 +524,25 @@ def test_get_modulemds_criteria(ubi_config):
     # let's not test internal structure of criteria, that's responsibility of pulplib
 
 
+def test_get_modulemd_defaults_criteria():
+    """Test proper creation of criteria for modulemd_defaults query"""
+    matcher = ModularMatcher(None, None)
+    unit = UbiUnit(
+        ModulemdUnit(
+            name="test", stream="10", version=100, context="abcd", arch="x86_64"
+        ),
+        None,
+    )
+    matcher.modules = [unit]
+    criteria = matcher._get_modulemd_defaults_criteria()
+    # there should be 1 criterium created based on modules list of Matcher obj.
+    assert len(criteria) == 1
+    # it should be instance of Criteria
+    for crit in criteria:
+        assert isinstance(crit, Criteria)
+    # let's not test internal structure of criteria, that's responsibility of pulplib
+
+
 def test_get_modular_srpms_criteria(ubi_config):
     """Testing creation of criteria for srpms query"""
     matcher = ModularMatcher(None, ubi_config.modules)
@@ -614,10 +667,17 @@ def test_modular_matcher_run(pulp, ubi_config):
             "test-7:1.0-1.x86_64.src",
         ],
     )
+
+    modulemd_defaults = ModulemdDefaultsUnit(
+        name="fake_name",
+        stream="fake_stream",
+        repo_id="binary_repo",
+        content_type_id="modulemd_defaults",
+    )
     pulp.insert_repository(repo_1)
     pulp.insert_repository(repo_2)
     pulp.insert_repository(repo_3)
-    pulp.insert_units(repo_1, [unit_1, modulemd])
+    pulp.insert_units(repo_1, [unit_1, modulemd, modulemd_defaults])
     pulp.insert_units(repo_2, [unit_2])
     pulp.insert_units(repo_3, [unit_3])
 
@@ -627,6 +687,7 @@ def test_modular_matcher_run(pulp, ubi_config):
 
     # each public attribute is properly set with one unit
     assert len(matcher.modules) == 1
+    assert len(matcher.module_defaults) == 1
     assert len(matcher.binary_rpms) == 1
     assert len(matcher.debug_rpms) == 1
     assert len(matcher.source_rpms) == 1
@@ -636,6 +697,10 @@ def test_modular_matcher_run(pulp, ubi_config):
     assert output_module.nsvca == "fake_name:fake_stream:100:abcd:x86_64"
     assert output_module.associate_source_repo_id == "binary_repo"
 
+    output_modulemd_defaults = matcher.module_defaults.pop()
+    assert output_modulemd_defaults.name == "fake_name"
+    assert output_modulemd_defaults.stream == "fake_stream"
+
     rpm = matcher.binary_rpms.pop()
     assert rpm.filename == "test-1.0-1.x86_64.x86_64.rpm"
     assert rpm.associate_source_repo_id == "binary_repo"

tests/test_pulp.py

@@ -229,7 +229,6 @@ def test_search_module_defaults(mock_pulp, mock_search_module_defaults, mock_rep
     assert len(found_module_defaults) == 1
     assert found_module_defaults[0].name == "virt"
     assert found_module_defaults[0].stream == "rhel"
-    assert found_module_defaults[0].name_profiles == "virt:[rhel:common,default]"
 
 
 @pytest.fixture(name="search_task_response")

tests/test_ubipop.py

@@ -16,6 +16,7 @@
     Distributor,
     ModulemdUnit,
     RpmUnit,
+    ModulemdDefaultsUnit,
 )
 from mock import MagicMock, patch, call
 from more_executors import Executors
@@ -408,7 +409,6 @@ def test_match_module_defaults(mock_ubipop_runner):
     md_d = mock_ubipop_runner.repos.module_defaults["virtrhel"]
     assert len(md_d) == 1
     assert md_d[0].name == "virt"
-    assert md_d[0].name_profiles == "virt:[2.5:common]"
 
 
 def test_diff_modules(mock_ubipop_runner):
@@ -608,13 +608,18 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content_ft):
         f_return(set([get_test_mod(name="test_md", pulplib=True)]))
     )
 
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="test_mdd", stream="rhel", profiles={"2.5": "uncommon"}
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="test_mdd",
+                stream="rhel",
+                profiles={"2.5": "uncommon"},
+                repo_id="foo-rpms",
+                content_type_id="modulemd_defaults",
             ),
-        ],
-    }
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(
@@ -766,13 +771,19 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content_ft
     mock_ubipop_runner.repos.modules = f_proxy(
         f_return(set([get_test_mod(name="md_current", pulplib=True)]))
     )
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="mdd_current", stream="rhel", profiles={"2.5": "common"}
+
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="mdd_current",
+                stream="rhel",
+                profiles={"2.5": "common"},
+                repo_id="foo-rpms",
+                content_type_id="modulemd_defaults",
             ),
-        ],
-    }
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(
@@ -906,13 +917,19 @@ def test_get_pulp_no_duplicates(mock_ubipop_runner, mock_current_content_ft):
     mock_ubipop_runner.repos.modules = f_proxy(
         f_return(set([get_test_mod(name="md_current", pulplib=True)]))
     )
-    mock_ubipop_runner.repos.module_defaults = {
-        "test": [
-            get_test_mod_defaults(
-                name="mdd_current", stream="rhel", profiles={"2.5": "common"}
-            )
-        ]
-    }
+
+    mock_ubipop_runner.repos.module_defaults = [
+        UbiUnit(
+            ModulemdDefaultsUnit(
+                name="mdd_current",
+                stream="rhel",
+                profiles={"2.5": "common"},
+                repo_id="foo-rpms",
+                content_type_id="modulemd_defaults",
+            ),
+            "foo-rpms",
+        )
+    ]
 
     binary_rpms = [
         UbiUnit(

tests/test_utils.py

@@ -1,7 +1,7 @@
 import pytest
 
 from mock import MagicMock
-from pubtools.pulplib import YumRepository, RpmUnit
+from pubtools.pulplib import YumRepository, RpmUnit, ModulemdDefaultsUnit
 from ubipop._utils import (
     AssociateAction,
     AssociateActionModuleDefaults,
@@ -12,6 +12,7 @@
     UnassociateActionModules,
     UnassociateActionRpms,
     vercmp_sort,
+    flatten_md_defaults_name_profiles,
 )
 from ubipop._matcher import UbiUnit
 
@@ -121,3 +122,20 @@ def test_vercmp_sort():
     assert (unit_1 >= unit_2) is False
     assert (unit_1 > unit_2) is False
     assert (unit_1 != unit_2) is True
+
+
+def test_flatten_md_defaults_name_profiles():
+    unit = UbiUnit(
+        ModulemdDefaultsUnit(
+            content_type_id="modulemd_defaults",
+            name="test",
+            stream="foo",
+            profiles={"rhel": ["common", "uncommon"], "fedora": ["super", "ultra"]},
+            repo_id="foo-repo",
+        ),
+        "foo-repo",
+    )
+
+    out = flatten_md_defaults_name_profiles(unit)
+
+    assert out == "test:[fedora:super,ultra]:[rhel:common,uncommon]"

ubipop/__init__.py

@@ -5,6 +5,7 @@
 from collections import defaultdict, deque, namedtuple
 from concurrent.futures import as_completed
 from itertools import chain
+from attr import attr
 from pubtools.pulplib import Client, Criteria, PublishOptions
 
 import ubiconfig
@@ -20,6 +21,7 @@
     UnassociateActionModules,
     UnassociateActionModuleDefaults,
     UnassociateActionRpms,
+    flatten_md_defaults_name_profiles,
 )
 from ._matcher import ModularMatcher, RpmMatcher
 
@@ -394,9 +396,8 @@ def _get_pulp_actions_mds(self, modules, current):
         )
 
     def _get_pulp_actions_md_defaults(self, module_defaults, current):
-        module_defaults_list = list(chain.from_iterable(module_defaults.values()))
         return self._determine_pulp_actions(
-            module_defaults_list,
+            module_defaults,
             current,
             self._diff_md_defaults_by_profiles,
         )
@@ -503,15 +504,20 @@ def _diff_modules_by_nsvca(self, modules_1, modules_2):
 
     def _diff_md_defaults_by_profiles(self, module_defaults_1, module_defaults_2):
         return self._diff_lists_by_attr(
-            module_defaults_1, module_defaults_2, "name_profiles"
+            module_defaults_1, module_defaults_2, flatten_md_defaults_name_profiles
         )
 
     def _diff_packages_by_filename(self, packages_1, packages_2):
         return self._diff_lists_by_attr(packages_1, packages_2, "filename")
 
-    def _diff_lists_by_attr(self, list_1, list_2, attr):
-        attrs_list_2 = [getattr(obj, attr) for obj in list_2]
-        diff = [obj for obj in list_1 if getattr(obj, attr) not in attrs_list_2]
+    def _diff_lists_by_attr(self, list_1, list_2, attr_or_func):
+        def diff_attr(obj):
+            if callable(attr_or_func):
+                return attr_or_func(obj)
+            return getattr(obj, attr_or_func)
+
+        attrs_list_2 = [diff_attr(obj) for obj in list_2]
+        diff = [obj for obj in list_1 if diff_attr(obj) not in attrs_list_2]
 
         return diff
 
@@ -529,12 +535,11 @@ def run_ubi_population(self):
         rm = RpmMatcher(self.repos.in_repos, self.ubiconfig).run()
 
         self.repos.modules = mm.modules
+        self.repos.module_defaults = mm.module_defaults
         self.repos.packages = rm.binary_rpms
         self.repos.debug_rpms = rm.debug_rpms
         self.repos.source_rpms = rm.source_rpms
 
-        self._match_module_defaults()
-
         (
             associations,
             unassociations,

ubipop/_matcher.py

@@ -157,6 +157,11 @@ def _search_srpms(self, or_criteria, repos, batch_size_override=None):
     def _search_moludemds(self, or_criteria, repos):
         return self._search_units_per_repos(or_criteria, repos, content_type="modulemd")
 
+    def _search_modulemd_defaults(self, or_criteria, repos):
+        return self._search_units_per_repos(
+            or_criteria, repos, content_type="modulemd_defaults"
+        )
+
     def _get_srpms_criteria(self):
         filenames = []
         for rpms_list in as_completed([self.binary_rpms, self.debug_rpms]):
@@ -176,6 +181,7 @@ class ModularMatcher(Matcher):
     def __init__(self, input_repos, ubi_config):
         super(ModularMatcher, self).__init__(input_repos, ubi_config)
         self.modules = None
+        self.module_defaults = None
 
     def run(self):
         """Asynchronously creates criteria for pulp queries and
@@ -212,6 +218,16 @@ def run(self):
                 self._search_srpms, srpms_criteria, self._input_repos.source
             )
         )
+        modulemd_defaults_criteria = f_proxy(
+            self._executor.submit(self._get_modulemd_defaults_criteria)
+        )
+        self.module_defaults = f_proxy(
+            self._executor.submit(
+                self._search_modulemd_defaults,
+                modulemd_defaults_criteria,
+                self._input_repos.rpm,
+            )
+        )
         return self
 
     def _get_modular_rpms_criteria(self):
@@ -221,8 +237,14 @@ def _get_modular_rpms_criteria(self):
         return pkgs_or_criteria
 
     def _get_modulemds_criteria(self):
+        return self._get_criteria_for_modules(self._ubi_config)
+
+    def _get_modulemd_defaults_criteria(self):
+        return self._get_criteria_for_modules(self.modules)
+
+    def _get_criteria_for_modules(self, modules):
         criteria_values = []
-        for module in self._ubi_config:
+        for module in modules:
             criteria_values.append(
                 (
                     module.name,

ubipop/_pulp_client.py

@@ -355,15 +355,3 @@ def __init__(self, name, stream, profiles, src_repo_id):
 
     def __str__(self):
         return self.name
-
-    @property
-    def name_profiles(self):
-        """
-        flatten the profles and prepend name
-        format: name:[key:profile,profile]:[key:profile]
-        'ruby:[2.5:common,unique]'
-        """
-        result = self.name
-        for key in sorted(self.profiles):
-            result += ":[%s:%s]" % (key, ",".join(sorted(self.profiles[key])))
-        return result