Adds rsync distributor

This commit adds the docker rsync distributor. Allows docker web distributor to have an rsync
predistributor.

closes #1887
https://pulp.plan.io/issues/1887

common/pulp_docker/common/error_codes.py

@@ -23,3 +23,5 @@
                              "%(reason)s"),
                 ['repo', 'registry', 'reason'])
 DKR1008 = Error("DKR1008", _("Could not find registry API at %(registry)s"), ['registry'])
+DKR1009 = Error("DKR1009", _("Docker rsync distributor configuration requires a "
+                             "postdistributor_id."), [])

docs/tech-reference/distributor.rst

@@ -160,3 +160,83 @@ Example Redirect File Contents::
     ],
   "tags": {"latest": "769b9341d937a3dba9e460f664b4f183a6cecdd62b337220a28b3deb50ee0a02"}
  }
+
+Docker rsync Distributor
+------------------------
+
+Purpose:
+--------
+The Docker rsync distributor publishes docker content to a remote server. The distributor uses
+rsync over ssh to perform the file transfer. Docker images (v1) are published into the root of
+the remote repository. Manifests (v2) are published into ``manifests`` directory and Blobs (v2) are
+published into ``blobs`` directory.
+
+The docker rsync distributor makes it easier to serve docker content on one server and run Crane on
+another server. It is recommended that the rsync distributor is used required to publish prior to
+publishing with the docker web distributor.
+
+Configuration
+=============
+Here is an example docker_rsync_distributor configuration:
+
+.. code-block:: json
+
+    {
+     "distributor_id": "my_docker_rsync_distributor",
+     "distributor_type_id": "docker_rsync_distributor",
+     "distributor_config": {
+        "remote": {
+            "auth_type": "publickey",
+            "ssh_user": "foo",
+            "ssh_identity_file": "/home/user/.ssh/id_rsa",
+            "host": "192.168.121.1",
+            "root": "/home/foo/pulp_root_dir"
+        },
+        "postdistributor_id": "docker_web_distributor_name_cli"
+     }
+    }
+
+
+``postdistributor_id``
+  The id of the docker_distributor_web associated with the same repository. The
+  ``repo-registry-id`` configured in the postdistributor will be used when generating tags list.
+   The docker web distributor associated with the same repository is required to have the
+   ``predistributor_id`` configured. ``postdistributor_id`` is a required config.
+
+The ``distributor_config`` contains a ``remote`` section with the following settings:
+
+``auth_type``
+  Two authentication methods are supported: ``publickey`` and ``password``.
+
+``ssh_user``
+  The ssh user for remote server.
+
+``ssh_identity_file``
+  The path to the private key to be used as the ssh identity file. When ``auth_type`` is
+  ``publickey`` this is a required config. The key has to be readable by user ``apache``.
+
+``ssh_password``
+  The password to be used for ``ssh_user`` on the remote server. ``ssh_password`` is required when
+  ``auth_type`` is 'password'.
+
+``host``
+  The hostname of the remote server.
+
+``root``
+  The absolute path to the remote root directory where all the data (content and published content)
+  lives. This is the remote equivalent to ``/var/lib/pulp``. The repo id is appended to the
+  ``root`` path to determine the location of published repository.
+
+Optional Configuration
+----------------------
+
+``content_units_only``
+  If true, the distributor will publish content units only (e.g. ``/var/lib/pulp/content``). The
+  symlinks of a published repository will not be rsynced.
+
+``delete``
+  If true, ``--delete`` is appended to the rsync command for symlinks and repodata so that any old
+  files no longer present in the local published directory are removed from the remote server.
+
+``remote_units_path``
+  The relative path from the ``root`` where unit files will live. Defaults to ``content/units``.

plugins/pulp_docker/plugins/distributors/configuration.py

@@ -2,7 +2,9 @@
 import re
 from urlparse import urlparse
 
+from pulp.plugins.rsync import configuration as rsync_config
 from pulp.server.config import config as server_config
+from pulp.server.db.model import Distributor
 from pulp.server.exceptions import PulpCodedValidationException
 
 from pulp_docker.common import constants, error_codes
@@ -62,6 +64,55 @@ def validate_config(config, repo):
     return True, None
 
 
+def validate_rsync_distributor_config(repo, config, config_conduit):
+    """
+    Performs validation of configuration that is standard for all rsync distributors. Then performs
+    extra validation needed for docker rsync.
+
+    :param repo:   metadata describing the repository to which the
+                   configuration applies
+    :type  repo:   pulp.plugins.model.Repository
+    :param config: Pulp configuration for the distributor
+    :type  config: pulp.plugins.config.PluginCallConfiguration
+    :param config_conduit: Configuration Conduit;
+    :type  config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit
+
+    :return: tuple comprised of a boolean indicating whether validation succeeded or failed and a
+             list of errors (if any)
+    :rtype: (bool, list of strings) or (bool, None)
+    :raises: PulpCodedValidationException if any validations failed
+    """
+    valid, errors = rsync_config.validate_config(repo, config, config_conduit)
+    if valid:
+        return validate_postdistributor(repo, config)
+    else:
+        return valid, errors
+
+
+def validate_postdistributor(repo, config):
+    """
+    Validates that the postdistributor_id is set and is valid for this repositotry.
+
+    :param repo:   metadata describing the repository to which the configuration applies
+    :type  repo:   pulp.plugins.model.Repository
+    :param config: Pulp configuration for the distributor
+    :type  config: pulp.plugins.config.PluginCallConfiguration
+
+    :return: tuple comprised of a boolean indicating whether validation succeeded or failed and a
+             list of errors (if any)
+    :rtype: (bool, list of strings) or (bool, None)
+    :raises: PulpCodedValidationException if postdistributor_id is not defined or 404 if the
+             distributor_id is not associated with the repo
+
+    """
+    postdistributor = config.flatten().get("postdistributor_id", None)
+    if postdistributor:
+        Distributor.objects.get_or_404(repo_id=repo.id, distributor_id=postdistributor)
+        return True, None
+    else:
+        raise PulpCodedValidationException(error_code=error_codes.DKR1009)
+
+
 def get_root_publish_directory(config, docker_api_version):
     """
     The publish directory for the docker plugin
@@ -110,7 +161,6 @@ def get_web_publish_dir(repo, config, docker_api_version):
     :return: the HTTP publication directory
     :rtype:  str
     """
-
     return os.path.join(get_root_publish_directory(config, docker_api_version), 'web',
                         get_repo_relative_path(repo, config))