Enable storing RPM dependencies

Each RPM can have arbitrary number of dependencies of multiple types.
The types are hard-coded, and can not be extended at run-time.

With this patch, it is possible to store and retrieve the dependencies.
There are tests for these parts.

Existing tests have been updated to pass: for RPM tests there are some
minor updates. The compose/package API and its variants are updated not
to include dependencies in their output.

All changes to dependencies are logged in change sets as single update
to the RPM.

There is a single filter for each dependency type. It allows to specify
an optional version. When a version is used in the filter, it removes
all packages that are incompatible with that version restriction. The
behaviour is documented with examples.

The version comparisons are implemented with a function adhering to PEP
0440 [0], which seems to be powerful enough to do everything required.
There is a wrapper that makes it work with epochs. It also works with
releases.

There is an additional single filter `has_no_deps`, which filters
packages that have some or don't have any dependency (this works across
all types).

[0]: https://www.python.org/dev/peps/pep-0440/

Changes in other parts of codebase:

 * Custom boolean filter is updated to allow removing duplicates from the
   response.

JIRA: PDC-955

pdc/apps/common/filters.py

@@ -228,8 +228,11 @@ def filter(self, qs, value):
         if not value:
             return qs
         if value.lower() in self.TRUE_STRINGS:
-            return qs.filter(**{self.name: True})
+            qs = qs.filter(**{self.name: True})
         elif value.lower() in self.FALSE_STRINGS:
-            return qs.filter(**{self.name: False})
+            qs = qs.filter(**{self.name: False})
         else:
-            return qs.none()
+            qs = qs.none()
+        if self.distinct:
+            qs = qs.distinct()
+        return qs

pdc/apps/common/hacks.py

@@ -4,12 +4,15 @@
 # Licensed under The MIT License (MIT)
 # http://opensource.org/licenses/MIT
 #
+import re
+
 from django.db import connection
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from rest_framework import serializers
 
 from productmd import composeinfo, images, rpms
+from pkg_resources import parse_version
 
 
 def composeinfo_from_str(data):
@@ -113,3 +116,13 @@ def srpm_name_to_component_names(srpm_name):
         return binding_models.ReleaseComponentSRPMNameMapping.get_component_names_by_srpm_name(srpm_name)
     else:
         return [srpm_name]
+
+
+def parse_epoch_version(version):
+    """
+    Wrapper around `pkg_resources.parse_version` that can handle epochs
+    delimited by colon as is customary for RPMs.
+    """
+    if re.match(r'^\d+:', version):
+        version = re.sub(r'^(\d+):', r'\1!', version)
+    return parse_version(version)

pdc/apps/compose/views.py

@@ -1148,7 +1148,9 @@ def _packages_output(self, rpms):
         Output packages with unicode or dict
         """
         packages = [unicode(rpm) for rpm in rpms]
-        return packages if not self.to_dict else [RPMSerializer(rpm).data for rpm in rpms]
+        return (packages
+                if not self.to_dict
+                else [RPMSerializer(rpm, exclude_fields=['dependencies']).data for rpm in rpms])
 
     def _get_query_param_or_false(self, request, query_str):
         value = request.query_params.get(query_str)

pdc/apps/package/filters.py

@@ -3,16 +3,59 @@
 # Licensed under The MIT License (MIT)
 # http://opensource.org/licenses/MIT
 #
+from functools import partial
+
 from django.conf import settings
 from django.forms import SelectMultiple
+from django.core.exceptions import FieldError
 
 import django_filters
 
-from pdc.apps.common.filters import MultiValueFilter, MultiIntFilter, NullableCharFilter
-from pdc.apps.common.filters import CaseInsensitiveBooleanFilter
+from pdc.apps.common.filters import (MultiValueFilter, MultiIntFilter,
+                                     NullableCharFilter, CaseInsensitiveBooleanFilter)
 from . import models
 
 
+def dependency_filter(type, queryset, value):
+    m = models.Dependency.DEPENDENCY_PARSER.match(value)
+    if not m:
+        raise FieldError('Unrecognized value for filter for {}'.format(type))
+    groups = m.groupdict()
+    queryset = queryset.filter(dependency__name=groups['name'], dependency__type=type).distinct()
+    for dep in models.Dependency.objects.filter(type=type, name=groups['name']):
+
+        is_equal = dep.is_equal(groups['version']) if groups['version'] else False
+        is_lower = dep.is_lower(groups['version']) if groups['version'] else False
+        is_higher = dep.is_higher(groups['version']) if groups['version'] else False
+
+        if groups['op'] == '=' and not dep.is_satisfied_by(groups['version']):
+            queryset = queryset.exclude(pk=dep.rpm_id)
+
+        # User requests everything depending on higher than X
+        elif groups['op'] == '>' and dep.comparison in ('<', '<=', '=') and (is_lower or is_equal):
+            queryset = queryset.exclude(pk=dep.rpm_id)
+
+        # User requests everything depending on lesser than X
+        elif groups['op'] == '<' and dep.comparison in ('>', '>=', '=') and (is_higher or is_equal):
+            queryset = queryset.exclude(pk=dep.rpm_id)
+
+        # User requests everything depending on at least X
+        elif groups['op'] == '>=':
+            if dep.comparison == '<' and (is_lower or is_equal):
+                queryset = queryset.exclude(pk=dep.rpm_id)
+            elif dep.comparison in ('<=', '=') and is_lower:
+                queryset = queryset.exclude(pk=dep.rpm_id)
+
+        # User requests everything depending on at most X
+        elif groups['op'] == '<=':
+            if dep.comparison == '>' and (is_higher or is_equal):
+                queryset = queryset.exclude(pk=dep.rpm_id)
+            elif dep.comparison in ('>=', '=') and is_higher:
+                queryset = queryset.exclude(pk=dep.rpm_id)
+
+    return queryset
+
+
 class RPMFilter(django_filters.FilterSet):
     name        = MultiValueFilter()
     version     = MultiValueFilter()
@@ -25,11 +68,26 @@ class RPMFilter(django_filters.FilterSet):
     compose     = MultiValueFilter(name='composerpm__variant_arch__variant__compose__compose_id',
                                    distinct=True)
     linked_release = MultiValueFilter(name='linked_releases__release_id', distinct=True)
+    provides = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                          models.Dependency.PROVIDES))
+    requires = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                          models.Dependency.REQUIRES))
+    obsoletes = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                           models.Dependency.OBSOLETES))
+    conflicts = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                           models.Dependency.CONFLICTS))
+    recommends = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                            models.Dependency.RECOMMENDS))
+    suggests = django_filters.MethodFilter(action=partial(dependency_filter,
+                                                          models.Dependency.SUGGESTS))
+    has_no_deps = CaseInsensitiveBooleanFilter(name='dependency__isnull', distinct=True)
 
     class Meta:
         model = models.RPM
         fields = ('name', 'version', 'epoch', 'release', 'arch', 'srpm_name',
-                  'srpm_nevra', 'compose', 'filename', 'linked_release')
+                  'srpm_nevra', 'compose', 'filename', 'linked_release',
+                  'provides', 'requires', 'obsoletes', 'conflicts', 'recommends', 'suggests',
+                  'has_no_deps')
 
 
 class ImageFilter(django_filters.FilterSet):

pdc/apps/package/migrations/0005_auto_20150907_0905.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('package', '0004_rpm_linked_releases'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Dependency',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('type', models.PositiveIntegerField(choices=[(1, b'provides'), (2, b'requires'), (3, b'obsoletes'), (4, b'conflicts'), (5, b'recommends'), (6, b'suggests')])),
+                ('name', models.CharField(max_length=200)),
+                ('version', models.CharField(max_length=200, null=True, blank=True)),
+                ('comparison', models.CharField(max_length=50, null=True, blank=True)),
+                ('rpm', models.ForeignKey(to='package.RPM')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='dependency',
+            unique_together=set([('type', 'name', 'version', 'comparison', 'rpm')]),
+        ),
+    ]

pdc/apps/package/models.py

@@ -3,17 +3,18 @@
 # Licensed under The MIT License (MIT)
 # http://opensource.org/licenses/MIT
 #
+import re
+
 from django.db import models, connection, transaction
 from django.db.utils import IntegrityError
 from django.core.exceptions import ValidationError
 from django.forms.models import model_to_dict
 
-from pkg_resources import parse_version
 from kobo.rpmlib import parse_nvra
 
 from pdc.apps.common.models import get_cached_id
 from pdc.apps.common.validators import validate_md5, validate_sha1, validate_sha256
-from pdc.apps.common.hacks import add_returning
+from pdc.apps.common.hacks import add_returning, parse_epoch_version
 from pdc.apps.common.constants import ARCH_SRC
 from pdc.apps.release.models import Release
 
@@ -54,13 +55,16 @@ def check_srpm_nevra(srpm_nevra, arch):
             raise ValidationError("RPM's srpm_nevra should be empty if and only if arch is src")
 
     def export(self, fields=None):
-        _fields = set(['name', 'epoch', 'version', 'release', 'arch', 'filename',
-                       'srpm_name', 'srpm_nevra', 'linked_releases']) if fields is None else set(fields)
+        _fields = (set(['name', 'epoch', 'version', 'release', 'arch', 'filename',
+                        'srpm_name', 'srpm_nevra', 'linked_releases', 'dependencies'])
+                   if fields is None else set(fields))
         result = model_to_dict(self, fields=_fields - {'linked_releases'})
         if 'linked_releases' in _fields:
             result['linked_releases'] = []
             for linked_release in self.linked_releases.all():
                 result['linked_releases'].append(linked_release.release_id)
+        if 'dependencies' in _fields:
+            result['dependencies'] = self.dependencies
         return result
 
     @staticmethod
@@ -109,7 +113,113 @@ def bulk_insert(cursor, rpm_nevra, filename, srpm_nevra=None):
 
     @property
     def sort_key(self):
-        return (self.epoch, parse_version(self.version), parse_version(self.release))
+        return (self.epoch, parse_epoch_version(self.version), parse_epoch_version(self.release))
+
+    @property
+    def dependencies(self):
+        """
+        Get a dict with all deps of the RPM. All types of dependencies are
+        included.
+        """
+        result = {}
+        choices = dict(Dependency.DEPENDENCY_TYPE_CHOICES)
+        for type in choices.values():
+            result[type] = []
+        for dep in Dependency.objects.filter(rpm=self):
+            result[choices[dep.type]].append(unicode(dep))
+        return result
+
+
+class Dependency(models.Model):
+    PROVIDES = 1
+    REQUIRES = 2
+    OBSOLETES = 3
+    CONFLICTS = 4
+    RECOMMENDS = 5
+    SUGGESTS = 6
+    DEPENDENCY_TYPE_CHOICES = (
+        (PROVIDES, 'provides'),
+        (REQUIRES, 'requires'),
+        (OBSOLETES, 'obsoletes'),
+        (CONFLICTS, 'conflicts'),
+        (RECOMMENDS, 'recommends'),
+        (SUGGESTS, 'suggests'),
+    )
+
+    DEPENDENCY_PARSER = re.compile(r'^(?P<name>[^ <>=]+)( *(?P<op>=|>=|<=|<|>) *(?P<version>[^ <>=]+))?$')
+
+    type = models.PositiveIntegerField(choices=DEPENDENCY_TYPE_CHOICES)
+    name = models.CharField(max_length=200)
+    version = models.CharField(max_length=200, blank=True, null=True)
+    comparison = models.CharField(max_length=50, blank=True, null=True)
+    rpm = models.ForeignKey(RPM)
+
+    class Meta:
+        unique_together = (
+            ('type', 'name', 'version', 'comparison', 'rpm')
+        )
+
+    def __unicode__(self):
+        base_str = self.name
+        if self.version:
+            base_str += ' {comparison} {version}'.format(comparison=self.comparison,
+                                                         version=self.version)
+        return base_str
+
+    def clean(self):
+        """
+        When version constraint is set, both a version and comparison type must
+        be specified.
+        """
+        if (self.version is None) != (self.comparison is None):
+            # This code should be unreachable based on user input, and only
+            # programmer error can cause this to fail.
+            raise ValidationError('Bad version constraint: both version and comparison must be specified.')
+
+    @property
+    def parsed_version(self):
+        if not hasattr(self, '_version'):
+            self._version = parse_epoch_version(self.version)
+        return self._version
+
+    def is_satisfied_by(self, other):
+        """
+        Check if other version satisfies this dependency.
+
+        :paramtype other: string
+        """
+        funcs = {
+            '=': lambda x: x == self.parsed_version,
+            '<': lambda x: x < self.parsed_version,
+            '<=': lambda x: x <= self.parsed_version,
+            '>': lambda x: x > self.parsed_version,
+            '>=': lambda x: x >= self.parsed_version,
+        }
+        return funcs[self.comparison](parse_epoch_version(other))
+
+    def is_equal(self, other):
+        """
+        Return true if the other version is equal to version in this dep.
+
+        :paramtype other: string
+        """
+        return self.parsed_version == parse_epoch_version(other)
+
+    def is_higher(self, other):
+        """
+        Return true if version in this dep is higher than the other version.
+
+        :paramtype other: string
+        """
+        return self.parsed_version > parse_epoch_version(other)
+
+    def is_lower(self, other):
+        """
+        Return true if version in this dep is lower than the other version.
+
+        :paramtype other: string
+        """
+        return self.parsed_version < parse_epoch_version(other)
 
 
 class ImageFormat(models.Model):