Do not blacklist pkgs from modules

This change include also reworked finalizing
output pkgs set - where do not include modular package
which is not referenced by any module.

tests/test_pulp.py

@@ -45,7 +45,7 @@ def mock_response_for_async_req():
 @pytest.fixture()
 def search_rpms_response():
     yield [{"metadata": {"name": "foo-pkg", "filename": "foo-pkg.rpm",
-                         "sourcerpm": "foo-pkg.src.rpm"}}]
+                         "sourcerpm": "foo-pkg.src.rpm", "is_modular": False}}]
 
 
 @pytest.fixture()
@@ -131,6 +131,7 @@ def test_search_rpms(mock_pulp, mock_search_rpms, mock_repo):
     assert found_rpms[0].name == "foo-pkg"
     assert found_rpms[0].filename == "foo-pkg.rpm"
     assert found_rpms[0].sourcerpm_filename == "foo-pkg.src.rpm"
+    assert found_rpms[0].is_modular is False
 
 
 def test_search_modules(mock_pulp, mock_search_modules, mock_repo):

tests/test_ubipop.py

@@ -57,7 +57,8 @@ def get_test_repo(**kwargs):
 
 
 def get_test_pkg(**kwargs):
-    return Package(kwargs.get('name'), kwargs.get('filename'), kwargs.get('sourcerpm_filename'))
+    return Package(kwargs.get('name'), kwargs.get('filename'), kwargs.get('sourcerpm_filename'),
+                   kwargs.get('is_modular', False))
 
 
 def get_test_mod(**kwargs):
@@ -245,19 +246,22 @@ def test_keep_n_newest_packages(mock_ubipop_runner):
 
 def test_keep_n_newest_packages_with_referenced_pkg_in_module(mock_ubipop_runner):
     packages = [
-        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm"),
-        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm"),
-        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.8-1.el8+1944+b6c8e16f.noarch.rpm")]
+        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.6-1.el8+1944+b6c8e16f.noarch.rpm",
+                     is_modular=True),
+        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm",
+                     is_modular=True),
+        get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.8-1.el8+1944+b6c8e16f.noarch.rpm",
+                     is_modular=True)]
 
     mock_ubipop_runner.repos.modules["ns"] = []
     mock_ubipop_runner.repos.pkgs_from_modules["ns"] = \
-        [get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm")]
+        [get_test_pkg(name="tomcatjss", filename="tomcatjss-7.3.7-1.el8+1944+b6c8e16f.noarch.rpm",
+                      is_modular=True)]
 
     mock_ubipop_runner.keep_n_latest_packages(packages)
 
-    assert len(packages) == 2
-    assert "7.3.8" in packages[0].filename
-    assert "7.3.7" in packages[1].filename
+    assert len(packages) == 1
+    assert "7.3.7" in packages[0].filename
 
 
 @pytest.mark.parametrize("rhel_repo_set, ubi_repo_set, fail",
@@ -585,5 +589,6 @@ def test_exclude_blacklisted_packages(mock_ubipop_runner):
     mock_ubipop_runner._exclude_blacklisted_packages()
 
     assert len(mock_ubipop_runner.repos.packages) == 0
-    assert len(mock_ubipop_runner.repos.pkgs_from_modules) == 0
+    # no blacklisting from pkgs from mds
+    assert len(mock_ubipop_runner.repos.pkgs_from_modules) == 1
     assert len(mock_ubipop_runner.repos.debug_rpms) == 0

ubipop/__init__.py

@@ -243,11 +243,24 @@ def _exclude_blacklisted_packages(self):
             list(chain.from_iterable(self.repos.debug_rpms.values())))
 
         for pkg in blacklisted_binary:
-            self.repos.packages.pop(pkg.name, None)
-            self.repos.pkgs_from_modules.pop(pkg.name, None)
+            # blacklist only non-modular pkgs
+            self.repos.packages[pkg.name][:] = [_pkg for _pkg in
+                                                self.repos.packages.get(pkg.name, [])
+                                                if _pkg.is_modular]
+
+            # if there is nothing left, remove whole entry for package
+            if not self.repos.packages[pkg.name]:
+                self.repos.packages.pop(pkg.name, None)
 
         for pkg in blacklisted_debug:
-            self.repos.debug_rpms.pop(pkg.name, None)
+            # blacklist only non-modular debug pkgs
+            self.repos.debug_rpms[pkg.name][:] = [_pkg for _pkg in
+                                                  self.repos.debug_rpms.get(pkg.name, [])
+                                                  if _pkg.is_modular]
+
+            # if there is nothing left, remove whole entry for debug package
+            if not self.repos.debug_rpms[pkg.name]:
+                self.repos.debug_rpms.pop(pkg.name, None)
 
     def _finalize_modules_output_set(self):
         for _, modules in self.repos.modules.items():
@@ -549,23 +562,27 @@ def get_packages_from_module(self, package_name, input_modules):
                 if arch == 'src':
                     continue
                 if name == package_name:
-                    rpms.append(Package(name, rpm_without_epoch + '.rpm'))
+                    rpms.append(Package(name, rpm_without_epoch + '.rpm', is_modular=True))
 
         return rpms
 
     def keep_n_latest_packages(self, packages, n=1):
         """
-        Keep n latest packages,
-        package is deleted from output set if it's not referenced by any remaining module
+        Keep n latest packages non-modular,
+        modular packages are kept only if they are referenced by any remaining module,
         """
-        packages_to_delete = packages[:-n]
-
         packages_to_keep = []
-        for package in packages_to_delete:
-            for module_name_stream, packages_ref_by_module in \
-                    self.repos.pkgs_from_modules.items():
-                if package.filename in [pkg.filename for pkg in packages_ref_by_module] and\
-                                        module_name_stream in self.repos.modules:
-                    packages_to_keep.append(package)
-
-        packages[:] = packages[-n:] + packages_to_keep
+        non_modular_pkgs = []
+        for package in packages:
+            if package.is_modular:
+                for module_name_stream, packages_ref_by_module in \
+                        self.repos.pkgs_from_modules.items():
+                    pkgs_filenames_from_modules = [pkg.filename for pkg in packages_ref_by_module]
+                    if package.filename in pkgs_filenames_from_modules \
+                            and module_name_stream in self.repos.modules:
+                        # this skips modular pkgs that are not referenced by module
+                        packages_to_keep.append(package)
+            else:
+                non_modular_pkgs.append(package)
+
+        packages[:] = non_modular_pkgs[-n:] + packages_to_keep

ubipop/_pulp_client.py

@@ -81,7 +81,8 @@ def search_rpms(self, repo, name=None, arch=None, name_globbing=False):
         ret.raise_for_status()
         for item in ret.json():
             metadata = item['metadata']
-            rpms.append(Package(metadata['name'], metadata['filename'], metadata.get('sourcerpm')))
+            rpms.append(Package(metadata['name'], metadata['filename'], metadata.get('sourcerpm'),
+                                metadata.get('is_modular', False)))
         return rpms
 
     def search_modules(self, repo, name=None, stream=None):
@@ -225,10 +226,11 @@ def __init__(self, repo_id, arch, platform_full_version, distributors_ids_type_i
 
 
 class Package(object):
-    def __init__(self, name, filename, sourcerpm_filename=None):
+    def __init__(self, name, filename, sourcerpm_filename=None, is_modular=False):
         self.name = name
         self.filename = filename
         self.sourcerpm_filename = sourcerpm_filename
+        self.is_modular = is_modular
 
     def __lt__(self, other):
         return vercmp(self.filename, other.filename) < 0