Implement the deduplication of RpmRequiresToken issue #147

This allows us to remove some of the duplicated dependencies quite
easily.

spec_cleaner/rpmhelpers.py

@@ -194,13 +194,19 @@ def add_group(group):
     """
     Flatten the lines of the group from sublits to one simple list
     """
-    if isinstance(group, str) or isinstance(group, RpmRequiresToken):
+    if isinstance(group, str):
         return [group]
+    elif isinstance(group, RpmRequiresToken):
+        items = []
+        if group.comments:
+            items += group.comments
+        items.append(group)
+        return items
     elif isinstance(group, list):
-        x = []
+        items = []
         for subgroup in group:
-            x += add_group(subgroup)
-        return x
+            items += add_group(subgroup)
+        return items
     else:
         raise RpmException('Unknown type of group in preamble: %s' % type(group))
 

spec_cleaner/rpmpreamble.py

@@ -68,11 +68,6 @@ def __init__(self, options):
         self.allowed_groups = options['allowed_groups']
         # start the object
         self.paragraph = RpmPreambleElements(options)
-        # initialize list of groups that need to pass over conversion fixer
-        self.categories_with_package_tokens = self.paragraph.categories_with_sorted_package_tokens[:]
-        # these packages actually need fixing after we sent the values to
-        # reorder them
-        self.categories_with_package_tokens.append('provides_obsoletes')
         # license handling
         self.subpkglicense = options['subpkglicense']
         # modname detection
@@ -276,7 +271,7 @@ def _add_line_value_to(self, category, value, key=None):
         """
         key = self.paragraph.compile_category_prefix(category, key)
 
-        if category in self.categories_with_package_tokens:
+        if category in self.paragraph.categories_with_package_tokens:
             values = self._fix_list_of_packages(value, category)
             for value in values:
                 if isinstance(value, str):
@@ -290,8 +285,12 @@ def _add_line_value_to(self, category, value, key=None):
 
     def _add_line_to(self, category, line):
         if self.paragraph.current_group:
-            self.paragraph.current_group.append(line)
-            self.paragraph.items[category].append(self.paragraph.current_group)
+            if isinstance(line, RpmRequiresToken):
+                line.comments = self.paragraph.current_group
+                self.paragraph.items[category].append(line)
+            else:
+                self.paragraph.current_group.append(line)
+                self.paragraph.items[category].append(self.paragraph.current_group)
             self.paragraph.current_group = []
         else:
             self.paragraph.items[category].append(line)

spec_cleaner/rpmpreambleelements.py

@@ -113,6 +113,11 @@ def __init__(self, options):
         self.license = options['license']
         # dict of license replacement options
         self.license_conversions = options['license_conversions']
+        # initialize list of groups that need to pass over conversion fixer
+        self.categories_with_package_tokens = self.categories_with_sorted_package_tokens[:]
+        # these packages actually need fixing after we sent the values to
+        # reorder them
+        self.categories_with_package_tokens.append('provides_obsoletes')
 
     def _sort_helper_key(self, a):
         if isinstance(a, str) or isinstance(a, RpmRequiresToken):
@@ -203,11 +208,54 @@ def _verify_prereq_message(self, elements):
 
         return elements
 
-    def _remove_duplicates(self):
+    def _remove_duplicates(self, elements):
         """
         Remove duplicate requires/buildrequires/etc
         """
-        pass
+        results = []
+        for element in elements:
+            match = False
+            # anything else than requirestoken
+            if not isinstance(element, RpmRequiresToken):
+                results.append(element)
+                continue
+            # no results stored yet
+            if not results:
+                results.append(element)
+                continue
+            # search already stored content
+            for index, item in enumerate(results):
+                # names and prefix must always match
+                if item.name == element.name and item.prefix == element.prefix:
+                    # do we have full match on everything
+                    if item.version == element.version and item.operator == element.operator:
+                        # append comment if needed only as we are 100% match
+                        if element.comments:
+                            tmp = results[index]
+                            if tmp.comments:
+                                tmp.comments += element.comments
+                            else:
+                                tmp.comments = element.comments
+                            results[index] = tmp
+                        match = True
+                        break
+                    # new one specifies version
+                    if not item.version and element.version:
+                        if item.comments:
+                            if element.comments:
+                                element.comments += item.comments
+                            else:
+                                element.comments = item.comments
+                        results[index] = element
+                        match = True
+                        break
+                    # for version determination which could be ommited one
+                    # must use rpm versionCompare to get same results
+                    # unfortunately it uses too many resources so we simply
+                    # leave this to the maintainer
+            if not match:
+                results.append(element)
+        return results
 
 
     def _run_global_list_operations(self, phase, elements):
@@ -262,11 +310,14 @@ def flatten_output(self, needs_license=False, nested = False):
         # add pkgconfig dep
         self._add_pkgconfig_buildrequires(nested)
         # remove duplicates
-        self._remove_duplicates()
+        for i in self.categories_with_package_tokens:
+            self.items[i] = self._remove_duplicates(self.items[i])
         for i in self.categories_order:
             sorted_list = []
+            if i in self.categories_with_sorted_package_tokens:
+                self.items[i].sort(key=self._sort_helper_key)
             # sort-out within the ordered groups based on the key
-            if i in self.categories_with_sorted_package_tokens + self.categories_with_sorted_keyword_tokens:
+            if i in self.categories_with_sorted_keyword_tokens:
                 self.items[i].sort(key=self._sort_helper_key)
                 self.items[i] = sort_uniq(self.items[i])
             # flatten the list from list of lists as no reordering is planned

spec_cleaner/rpmrequirestoken.py

@@ -14,6 +14,7 @@ class RpmRequiresToken(object):
     operator = None
     version = None
     prefix = None
+    comments = None
 
     def __init__(self, name, operator = None, version = None, prefix = None):
         self.prefix = prefix