Adds ISO rsync distributor

closes #1963
https://pulp.plan.io/issues/1963

docs/tech-reference/iso-rsync-distributor.rst

@@ -0,0 +1,77 @@
+=====================
+ISO rsync Distributor
+=====================
+
+Purpose:
+========
+The ISO rsync distributor publishes ISO content to a remote server. The distributor uses rsync over
+ssh to perform the file transfer.
+
+Configuration
+=============
+Here's an example of rpm_rsync_distributor configuration:
+
+.. code-block:: json
+
+    {
+     "distributor_id": "my_iso_rsync_distributor",
+     "distributor_type_id": "iso_rsync_distributor",
+     "distributor_config": {
+        "remote": {
+            "auth_type": "publickey",
+            "ssh_user": "foo",
+            "ssh_identity_file": "/home/user/.ssh/id_rsa",
+            "host": "192.168.121.1",
+            "root": "/home/foo/pulp_root_dir"
+        },
+        "predistributor_id": "my_iso_distributor",
+        }
+    }
+
+
+``predistributor_id``
+  The id of the iso_distributor associated with the same repository. The publish history of this
+  iso_distributor determines if the publish will be incremental.
+
+The ``distributor_config`` contains a ``remote`` section made up of the following settings:
+
+``auth_type``
+  Two authentication methods are supported: ``publickey`` and ``password``.
+
+``ssh_user``
+  ssh user for remote server
+
+``ssh_identity_file``
+  A path to the private key that will be used as identity file for ssh. When ``auth_type`` is
+  ``publickey`` this is a required config. The key has to be readable by user ``apache``.
+
+``ssh_password``
+  Password for ``ssh_user`` on remote server. Password is required when ``auth_type`` is 'password'.
+
+``host``
+  The hostname of the remote server.
+
+``root``
+  Absolute path to the remote root directory where all the data (content and published content)
+  lives. Remote equivalent to ``/var/lib/pulp``.
+
+Optional configuration
+----------------------
+
+``skip_fast_forward``
+  If true, the rsync distributor will publish all of the content of the repository. If false
+  (default), the publish is incremental when the predistributor's last publish was incremental.
+
+``content_units_only``
+  If true, the distributor will publish content units only (e.g. ``/var/lib/pulp/content``). The
+  symlinks of a published repository will not be rsynced.
+
+``skip_repodata``
+  If true, repodata will be omitted from the publish.
+
+``delete``
+  If true, ``--delete`` is appended to the rsync command for symlinks and repodata so that any old files no longer present in
+  the local published directory are removed from the remote server.
+
+``remote_units_path``
+  The relative path from the ``root`` where unit files should live. Defaults to ``content/units``.

plugins/pulp_rpm/plugins/db/models.py

@@ -1357,6 +1357,15 @@ def calculate_size(file_handle):
         # Calculate the size by seeking to the end to find the file size with tell()
         return file_utils.calculate_size(file_handle)
 
+    def get_symlink_name(self):
+        """
+        Provides the name that should be used when creating a symlink.
+        :return: file name as it appears in a published repository
+        :rtype: str
+        """
+        return self.name
+
+
 
 class ISOManifest(object):
     """

plugins/pulp_rpm/plugins/distributors/iso_rsync_distributor/distributor.py

@@ -0,0 +1,111 @@
+import logging
+import gettext
+
+from pulp.common.config import read_json_config
+from pulp.plugins.distributor import Distributor
+from pulp.plugins.rsync import configuration
+
+from pulp_rpm.plugins.distributors.iso_rsync_distributor import publish
+from pulp_rpm.common.ids import TYPE_ID_ISO
+
+_ = gettext.gettext
+
+_LOG = logging.getLogger(__name__)
+
+TYPE_ID_DISTRIBUTOR_ISO_RSYNC = 'iso_rsync_distributor'
+CONF_FILE_PATH = 'server/plugins.conf.d/%s.json' % TYPE_ID_DISTRIBUTOR_ISO_RSYNC
+
+DISTRIBUTOR_DISPLAY_NAME = 'ISO Rsync Distributor'
+
+
+def entry_point():
+    config = read_json_config(CONF_FILE_PATH)
+    return ISORsyncDistributor, config
+
+
+class ISORsyncDistributor(Distributor):
+    """
+    Distributor class for publishing RPM repo to remote server.
+
+    :ivar canceled: if true, task has been canceled
+    :ivar _publisher: instance of RPMRsyncPublisher
+    """
+
+    def __init__(self):
+        super(ISORsyncDistributor, self).__init__()
+
+        self.canceled = False
+        self._publisher = None
+
+    @classmethod
+    def metadata(cls):
+        """
+        Used by Pulp to classify the capabilities of this distributor.
+
+        :return: description of the distributor's capabilities
+        :rtype:  dict
+        """
+        return {'id': TYPE_ID_DISTRIBUTOR_ISO_RSYNC,
+                'display_name': DISTRIBUTOR_DISPLAY_NAME,
+                'types': [TYPE_ID_ISO]}
+
+    # -- repo lifecycle methods ------------------------------------------------
+
+    def validate_config(self, repo, config, config_conduit):
+        """
+        Allows the distributor to check the contents of a potential configuration
+        for the given repository. This call is made both for the addition of
+        this distributor to a new repository as well as updating the configuration
+        for this distributor on a previously configured repository.
+
+        :param repo: metadata describing the repository to which the
+                     configuration applies
+        :type  repo: pulp.plugins.model.Repository
+
+        :param config: plugin configuration instance; the proposed repo
+                       configuration is found within
+        :type  config: pulp.plugins.config.PluginCallConfiguration
+
+        :param config_conduit: Configuration Conduit;
+        :type  config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit
+
+        :return: tuple of (bool, str) to describe the result
+        :rtype:  tuple
+        """
+        _LOG.debug('Validating iso repository configuration: %s' % repo.id)
+
+        return configuration.validate_config(repo, config, config_conduit)
+
+    # -- actions ---------------------------------------------------------------
+
+    def publish_repo(self, repo, publish_conduit, config):
+        """
+        Publishes the given repository.
+
+        :param repo: metadata describing the repository
+        :type  repo: pulp.plugins.model.Repository
+
+        :param publish_conduit: provides access to relevant Pulp functionality
+        :type  publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
+
+        :param config: plugin configuration
+        :type  config: pulp.plugins.config.PluginConfiguration
+
+        :return: report describing the publish run
+        :rtype:  pulp.plugins.model.PublishReport
+        """
+        _LOG.debug('Publishing ISO repository: %s' % repo.id)
+
+        self._publisher = publish.ISORsyncPublisher(repo, publish_conduit, config,
+                                                    TYPE_ID_DISTRIBUTOR_ISO_RSYNC)
+        return self._publisher.publish()
+
+    def cancel_publish_repo(self):
+        """
+        Call cancellation control hook.
+        """
+        _LOG.debug('Canceling publishing repo to remote server')
+
+        self.canceled = True
+        if self._publisher is not None:
+            self._publisher.cancel()

plugins/pulp_rpm/plugins/distributors/iso_rsync_distributor/publish.py

@@ -0,0 +1,103 @@
+import os
+
+from pulp.plugins.rsync.publish import Publisher, RSyncPublishStep
+from pulp.plugins.util.publish_step import RSyncFastForwardUnitPublishStep
+from pulp.server.db.model import Distributor
+from pulp.server.exceptions import PulpCodedException
+
+from pulp_rpm.common import constants, ids
+from pulp_rpm.plugins import error_codes
+
+class ISORsyncPublisher(Publisher):
+
+    UNIT_FIELDS = ["_storage_path", "name"]
+
+    def _get_predistributor(self):
+        """
+        Returns the distributor that is configured as predistributor.
+        """
+        predistributor = self.get_config().flatten().get("predistributor_id", None)
+        if predistributor:
+            return Distributor.objects.get_or_404(repo_id=self.repo.id,
+                                                  distributor_id=predistributor)
+        else:
+            raise PulpCodedException(error_code=error_codes.RPM1011)
+
+    def _get_root_publish_dir(self):
+        """
+        Returns the publish directory path for the predistributor.
+
+        :return: absolute path to the master publish directory of predistributor
+        :rtype: str
+        """
+        if self.predistributor["config"].get("http", False):
+            return constants.ISO_HTTP_DIR
+        else:
+            return constants.ISO_HTTPS_DIR
+
+    def get_master_directory(self):
+        """
+        Returns path to master directory of the predistributor.
+
+        :return: path to 'master' publish directory
+        :rtype: str
+        """
+        return os.path.join(self._get_root_publish_dir(), self.repo.id)
+
+    def _add_necesary_steps(self, date_filter=None, config=None):
+        """
+        This method adds all the steps that are needed to accomplish an ISO rsync publish. This
+        includes:
+
+        Unit Query Step - selects units associated with the repo based on the date_filter and creates
+                          relative symlinks
+        Rsync Step (content units) - rsyncs units discovered in previous step to the remote server
+        Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server
+        Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server
+
+
+        :param date_filter:  Q object with start and/or end dates, or None if start and end dates
+                             are not provided
+        :type date_filter:  mongoengine.Q or types.NoneType
+        :param config: Pulp configuration for the distributor
+        :type  config: pulp.plugins.config.PluginCallConfiguration
+        """
+        remote_repo_path = self.repo.id
+
+        # Find all the units associated with repo before last publish with predistributor
+        gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)",
+                                                   repo=self.repo.repo_obj,
+                                                   unit_type=ids.TYPE_ID_ISO,
+                                                   date_filter=date_filter,
+                                                   unit_fields=ISORsyncPublisher.UNIT_FIELDS,
+                                                   remote_repo_path=remote_repo_path,
+                                                   published_unit_path=[])
+        self.add_child(gen_step)
+
+        dest_content_units_dir = self.get_units_directory_dest_path()
+        src_content_units_dir = self.get_units_src_path()
+
+
+        # Rsync content units
+        self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list,
+                                        src_content_units_dir, dest_content_units_dir,
+                                        config=config, exclude=[]))
+
+        # Stop here if distributor is only supposed to publish actual content
+        if self.get_config().flatten().get("content_units_only"):
+            return
+
+        # Rsync symlinks to the remote server
+        self.add_child(RSyncPublishStep("Rsync step (symlinks)",
+                                        self.symlink_list, self.symlink_src,
+                                        remote_repo_path,
+                                        config=config, links=True, exclude=["repodata"],
+                                        delete=self.config.get("force_refresh")))
+
+        predistributor_master_dir = self.get_master_directory()
+
+        # Rsync PULP_MANIFEST
+        self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)",
+                                        ['PULP_MANIFEST'], predistributor_master_dir,
+                                        remote_repo_path,
+                                        config=config))

plugins/pulp_rpm/plugins/error_codes.py

@@ -24,3 +24,4 @@
                 ['checksumtype'])
 RPM1009 = Error("RPM1009", _('Checksum type "%(checksumtype)s" is not supported.'),
                 ['checksumtype'])
+RPM1011 = Error("RPM1011", _('ISORsyncDistributor requires a predistributor to be configured.'), [])

plugins/setup.py

@@ -25,7 +25,9 @@
             'distributor = pulp_rpm.plugins.distributors.yum.distributor:entry_point',
             'ExportDistributor = pulp_rpm.plugins.distributors.export_distributor.distributor:'
             'entry_point',
-            'IsoDistributor = pulp_rpm.plugins.distributors.iso_distributor.distributor:entry_point'
+            'IsoDistributor = pulp_rpm.plugins.distributors.iso_distributor.distributor:entry_point',
+            'IsoRsyncDistributor = pulp_rpm.plugins.distributors.iso_rsync_distributor.distributor'
+            ':entry_point'
         ],
         'pulp.group_distributors': [
             'rpm_export = pulp_rpm.plugins.distributors.export_distributor.groupdistributor:'