Fix creation of source RPMs output set

Previously the method _get_rpm_output_set() stripped away
all modular packages and kept only the latest version of packages.

This seems to be too strict for source RPMs in some edge cases, at the
time when source RPMs output set is created, beacause the output set is
based on final output sets of binary and debug rpms that are already properly
filtered - they include only the latest versions of RPMs and non-modular
packages. Because of that we don't need to check again if the source RPM is
modular or not, or keep the latest version of SRPM. Only blacklisting
should happen. There are some edge cases when the original implementation
stripped some source RPMs that should be part of the output set. It's
probable that the edge cases happen due to data inconsistency especially
for modular packages.

We always need to include source RPMs of all binary and debug RPMs
unless they're blacklisted.

tests/test_matcher.py

@@ -825,6 +825,82 @@ def test_get_rpm_output_set(ubi_config):
     assert output[0].version == "11"
 
 
+def test_get_rpm_output_set_include_modular_and_all_versions(ubi_config):
+    """tests getting rpm output set from RpmMatcher, modular packages included
+    as well as all version of given package, only blacklist is applied"""
+    matcher = RpmMatcher(None, ubi_config)
+    # This unit will be in the output set - we allow all versions
+    unit_1 = UbiUnit(
+        RpmUnit(
+            name="test",
+            version="10",
+            release="20",
+            arch="x86_64",
+        ),
+        None,
+    )
+    # this one will be excluded using blacklist
+    unit_2 = UbiUnit(
+        RpmUnit(
+            name="excluded_with_globbing123456789",
+            version="11",
+            release="20",
+            arch="x86_64",
+        ),
+        None,
+    )
+    # this one will be excluded using blacklist
+    unit_3 = UbiUnit(
+        RpmUnit(
+            name="excluded_package",
+            version="10",
+            release="20",
+            arch="x86_64",
+        ),
+        None,
+    )
+    # this is another unit that will be in the output set
+    unit_4 = UbiUnit(
+        RpmUnit(
+            name="test",
+            version="11",
+            release="20",
+            arch="x86_64",
+        ),
+        None,
+    )
+    # this one is included, we don't exclude modular packages this time
+    unit_5 = UbiUnit(
+        RpmUnit(
+            name="modular_package",
+            version="12",
+            release="20",
+            arch="x86_64",
+            filename="modular_package.rpm",
+        ),
+        None,
+    )
+
+    rpms = [unit_1, unit_2, unit_3, unit_4, unit_5]
+    output = matcher._get_rpm_output_set(
+        rpms, modular_rpm_filenames=None, keep_all_versions=True
+    )
+
+    # in the output set there should be three units
+    assert len(output) == 3
+    # let's sort the output, order is not guranteed
+    output.sort(key=lambda x: x.version)
+
+    assert output[0].name == "test"
+    assert output[0].version == "10"
+
+    assert output[1].name == "test"
+    assert output[1].version == "11"
+
+    assert output[2].name == "modular_package"
+    assert output[2].version == "12"
+
+
 def test_get_pkgs_from_all_modules(pulp):
     """tests getting pkgs filenames from all available modulemd units"""
     repo = YumRepository(

ubipop/_matcher.py

@@ -354,9 +354,15 @@ def run(self):
             )
         )
 
+        # the output set of source rpms is almost ready at this point
+        # because it was created from final output set of binary and debug rpm
+        # so just need to apply blacklist and nothing else
         self.source_rpms = f_proxy(
             self._executor.submit(
-                self._get_rpm_output_set, source_rpms, modular_rpm_filenames
+                self._get_rpm_output_set,
+                source_rpms,
+                modular_rpm_filenames=None,
+                keep_all_versions=True,
             )
         )
 
@@ -393,7 +399,9 @@ def extract_modular_filenames():
         modules = self._search_moludemds([Criteria.true()], self._input_repos.rpm)
         return self._executor.submit(extract_modular_filenames)
 
-    def _get_rpm_output_set(self, rpms, modular_rpm_filenames):
+    def _get_rpm_output_set(
+        self, rpms, modular_rpm_filenames=None, keep_all_versions=False
+    ):
         blacklist_parsed = self._parse_blacklist_config()
         name_rpms_maps = {}
 
@@ -414,9 +422,10 @@ def is_blacklisted(rpm):
                     return blacklisted
 
         for rpm in rpms:
-            # skip modular rpms
-            if rpm.filename in modular_rpm_filenames:
-                continue
+            if modular_rpm_filenames:
+                # skip modular rpms
+                if rpm.filename in modular_rpm_filenames:
+                    continue
             # skip blacklisted rpms
             if is_blacklisted(rpm):
                 continue
@@ -426,8 +435,9 @@ def is_blacklisted(rpm):
         out = []
         # sort rpms and keep N latest versions of them
         for rpm_list in name_rpms_maps.values():
-            rpm_list.sort(key=vercmp_sort())
-            self._keep_n_latest_rpms(rpm_list)
+            if not keep_all_versions:
+                rpm_list.sort(key=vercmp_sort())
+                self._keep_n_latest_rpms(rpm_list)
             out.extend(rpm_list)
 
         return out