Introduce the Recommends dependency

Pulp has been ignoring the Recommends weak dependency when processing e.g
recursive unit copies.  This patch updates the RPM model to track a
recommends unit attribute and the primary.xml parser to populate this
attribute.  The libsolv dependency solver is used to process this
attribute when calculating unit dependencies.

Migration: populate the recommends attribute

Walks over the rpm units, unzipping the primary metadata XML snippets,
parsing the recommends entries and populating the recommends unit
attributes accoridingly.

Fixes: #3847
https://pulp.plan.io/issues/3847

docs/user-guide/release-notes/2.17.x.rst

@@ -24,6 +24,9 @@ New Features
 * More conservative dependency solving is now supported when e.g recursively
   copying RPM content between repositories.
 
+* The weak forward dependency `recommends` is now processed when e.g recursively
+  copying RPM content between repositories.
+
 * Advanced support for modular content is introduced. In addition to regular sync and publish
   of modules and it's defaults, it is possibile to upload them into Pulp by providing a yaml doc
   file. Copy and removal is also possible. To trigger recursive mode for modules, `recursive` option

plugins/pulp_rpm/plugins/db/models.py

@@ -711,6 +711,10 @@ class RpmBase(NonMetadataPackage):
                     with the "release", "epoch", "version", "flags", and "name" field.
     :type requires: list of dict
 
+    :ivar recommends: List of weak dependencies of a package. Each entry is a dictionary
+                      with a mandatory "name" field.
+    :type recommends: list of dict
+
     :ivar sourcerpm: Name of the source package (srpm) the package was built from.
     :type sourcerpm: mongoengine.StringField
 
@@ -770,6 +774,7 @@ class RpmBase(NonMetadataPackage):
     summary = mongoengine.StringField()
     time = mongoengine.IntField()
     requires = mongoengine.ListField()
+    recommends = mongoengine.ListField()
 
     unit_key_fields = ('name', 'epoch', 'version', 'release', 'arch', 'checksumtype', 'checksum')
 

plugins/pulp_rpm/plugins/importers/yum/pulp_solv.py

@@ -227,6 +227,7 @@ def unit_solvable_converter_factory(*attribute_factories):
     plain_attribute_factory('vendor'),
     rpm_dependency_attribute_factory('requires'),
     rpm_dependency_attribute_factory('provides'),
+    rpm_dependency_attribute_factory('recommends'),
     rpm_filelist_conversion,
 )
 

plugins/pulp_rpm/plugins/importers/yum/repomd/primary.py

@@ -42,6 +42,7 @@
 RPM_HEADER_RANGE_TAG = '{%s}header-range' % RPM_SPEC_URL
 RPM_PROVIDES_TAG = '{%s}provides' % RPM_SPEC_URL
 RPM_REQUIRES_TAG = '{%s}requires' % RPM_SPEC_URL
+RPM_RECOMMENDS_TAG = '{%s}recommends' % RPM_SPEC_URL
 RPM_ENTRY_TAG = '{%s}entry' % RPM_SPEC_URL
 
 # package information dictionary -----------------------------------------------
@@ -217,6 +218,11 @@ def _process_format_element(format_element):
         package_format['requires'] = \
             [_process_rpm_entry_element(e) for e in requires_element.findall(RPM_ENTRY_TAG)]
 
+    recommends_element = format_element.find(RPM_RECOMMENDS_TAG)
+    if recommends_element is not None:
+        package_format['recommends'] = \
+            [_process_rpm_entry_element(e) for e in recommends_element.findall(RPM_ENTRY_TAG)]
+
     package_format['files'] = \
         [_process_file_element(e) for e in format_element.findall(FILE_TAG)]
 

plugins/pulp_rpm/plugins/migrations/0043_populate_recommends.py

@@ -0,0 +1,71 @@
+import gzip
+import sys
+
+from pulp.server.db.connection import get_collection
+from pulp.server.db.migrations.lib import utils
+
+if sys.version_info < (2, 7):
+    from xml.etree import ElementTree as ET
+else:
+    from xml.etree import cElementTree as ET
+
+
+_NAMESPACES = {
+    'common': "http://linux.duke.edu/metadata/common",
+    'rpm': 'http://linux.duke.edu/metadata/rpm',
+}
+
+# The unit metadata snippets miss header/encoding and namespace references
+# which aren't included until publish time. This breaks the parsing.
+_HEADER = ('<?mxl version="1.0" encoding="UTF-8"?>\n'
+           '<metadata xmlns="http://linux.duke.edu/metadata/common" '
+           'xmlns:rpm="http://linux.duke.edu/metadata/rpm">\n {}'
+           '</metadata>\n')
+
+
+def migrate_rpm(collection, unit):
+    """
+    Uncompress single rpm unit primary metadata and populate the unit recommends attribute
+    accordingly.
+
+    :param collection: a collection of RPM units
+    :type collection: pymongo.collection.Collection
+    :param unit: the RPM unit being migrated
+    :type unit: dict
+    """
+    primary_xml = unit.get('repodata', {}).get('primary', '')
+    primary_xml = _HEADER.format(gzip.zlib.decompress(primary_xml))
+    root_element = ET.fromstring(primary_xml)
+    delta = {}
+    # the evr+flags fields are actually the attrib attribute of the entry node
+    # <rpm:entry name="foo" epoch="0" version="3.14" release="pi" flags="EQ" />
+    delta['recommends'] = [
+        rpm_entry.attrib for rpm_entry in root_element.iterfind(
+            './common:package/common:format/rpm:recommends/rpm:entry', _NAMESPACES)
+    ]
+    if delta['recommends']:
+        # NOTE(performance): update just in case non-empty recommends; empty or None recommends
+        # will be handled by the model
+        collection.update_one({'_id': unit['_id']}, {'$set': delta})
+
+
+def migrate(*args, **kwargs):
+    """
+    Populate the RPM unit recommends attribute.
+    Migration can be safely re-run multiple times.
+
+    :param args: unused
+    :type args: list
+    :param kwargs: unused
+    type kwargs: dict
+    """
+    rpm_collection = get_collection('units_rpm')
+    # select only units without the recommends attribute, fetch just the
+    # 'repodata.primary' attribute; the _id is always included
+    rpm_selection = rpm_collection.find(
+        {'recommends': {'$exists': False}}, ['repodata.primary']).batch_size(100)
+    total_rpm_units = rpm_selection.count()
+    with utils.MigrationProgressLog('RPM', total_rpm_units) as progress_log:
+        for rpm in rpm_selection:
+            migrate_rpm(rpm_collection, rpm)
+            progress_log.progress()

plugins/test/unit/plugins/migrations/test_0043_populate_recommends.py

@@ -0,0 +1,83 @@
+import unittest
+
+import mock
+
+from pulp.server.db.migrate.models import _import_all_the_way
+
+migration = _import_all_the_way('pulp_rpm.plugins.migrations.0043_populate_recommends')
+
+
+class TestMigrate(unittest.TestCase):
+    """
+    Test the migrate() function
+    """
+    @mock.patch.object(migration, 'utils')
+    @mock.patch.object(migration, 'migrate_rpm')
+    @mock.patch.object(migration, 'get_collection')
+    def test_calls_correct_functions(self, mock_get_collection, mock_migrate_rpm, mock_utils):
+        find_mock = mock_get_collection.return_value.find
+        batch_size_mock = find_mock.return_value.batch_size
+        # fake cursor
+        selection_mock = batch_size_mock.return_value = mock.MagicMock()
+        unit_mock = mock.MagicMock()
+        selection_mock.__iter__.return_value = [unit_mock]
+
+        migration.migrate()
+        mock_get_collection.assert_called_once_with('units_rpm')
+        find_mock.assert_called_once_with(
+            {'recommends': {'$exists': False}}, ['repodata.primary']
+        )
+        batch_size_mock.assert_called_once_with(100)
+        count_mock = batch_size_mock.return_value.count
+        count_mock.assert_called_once_with()
+        mock_utils.MigrationProgressLog.assert_called_once_with('RPM', count_mock.return_value)
+        progress_log_mock = mock_utils.MigrationProgressLog.return_value.__enter__.return_value
+        mock_migrate_rpm.assert_called_once_with(mock_get_collection.return_value, unit_mock)
+        progress_log_mock.progress.assert_called_once_with()
+
+
+class TestMigrateRpm(unittest.TestCase):
+    """
+    Test the migrate_rpm() function
+    """
+    def setUp(self):
+        super(TestMigrateRpm, self).setUp()
+        self.collection_mock = mock.Mock()
+        # fake a dict
+        self.unit_mock = mock.MagicMock()
+
+    @mock.patch.object(migration, 'ET')
+    @mock.patch.object(migration, 'gzip')
+    @mock.patch.object(migration, '_HEADER')
+    def test_calls_correct_functions(self, mock__HEADER, mock_gzip, mock_ET):
+        root_element_mock = mock_ET.fromstring.return_value
+
+        root_element_mock.iterfind.return_value = [
+            self.unit_mock,
+        ]
+        migration.migrate_rpm(self.collection_mock, self.unit_mock)
+        self.unit_mock.get.assert_called_once_with('repodata', {})
+        repodata_mock = self.unit_mock.get.return_value
+        repodata_mock.get.assert_called_once_with('primary', '')
+        primary_mock = repodata_mock.get.return_value
+        mock_gzip.zlib.decompress.assert_called_once_with(primary_mock)
+        decompress_mock = mock_gzip.zlib.decompress.return_value
+        mock__HEADER.format.assert_called_once_with(decompress_mock)
+        primary_xml_mock = mock__HEADER.format.return_value
+        mock_ET.fromstring.assert_called_once_with(primary_xml_mock)
+        root_element_mock.iterfind.assert_called_once_with(
+            './common:package/common:format/rpm:recommends/rpm:entry', migration._NAMESPACES
+        )
+        self.collection_mock.update_one.assert_called_once_with(
+            {'_id': self.unit_mock['_id']},
+            {'$set': {'recommends': [self.unit_mock.attrib]}}
+        )
+
+    @mock.patch.object(migration, 'ET')
+    @mock.patch.object(migration, 'gzip')
+    @mock.patch.object(migration, '_HEADER')
+    def test_no_update(self, _, __, mock_ET):
+        root_element_mock = mock_ET.fromstring.return_value
+        root_element_mock.iterfind.return_value = []
+        migration.migrate_rpm(self.collection_mock, self.unit_mock)
+        self.collection_mock.update_one.assert_not_called()