-
Notifications
You must be signed in to change notification settings - Fork 119
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #940 from dkliban/iso-rsync-distributor
Adds ISO rsync distributor
- Loading branch information
Showing
7 changed files
with
298 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
===================== | ||
ISO rsync Distributor | ||
===================== | ||
|
||
Purpose: | ||
======== | ||
The ISO rsync distributor publishes ISO content to a remote server. The distributor uses rsync over | ||
ssh to perform the file transfer. | ||
|
||
Configuration | ||
============= | ||
Here is an example iso_rsync_distributor configuration: | ||
|
||
.. code-block:: json | ||
{ | ||
"distributor_id": "my_iso_rsync_distributor", | ||
"distributor_type_id": "iso_rsync_distributor", | ||
"distributor_config": { | ||
"remote": { | ||
"auth_type": "publickey", | ||
"ssh_user": "foo", | ||
"ssh_identity_file": "/home/user/.ssh/id_rsa", | ||
"host": "192.168.121.1", | ||
"root": "/home/foo/pulp_root_dir" | ||
}, | ||
"predistributor_id": "my_iso_distributor", | ||
} | ||
} | ||
``predistributor_id`` | ||
The id of the iso_distributor associated with the same repository. The PULP_MANIFEST published by | ||
the predistributor is copied to the remote server. | ||
|
||
The ``distributor_config`` contains a ``remote`` section with the following settings: | ||
|
||
``auth_type`` | ||
Two authentication methods are supported: ``publickey`` and ``password``. | ||
|
||
``ssh_user`` | ||
The ssh user for remote server. | ||
|
||
``ssh_identity_file`` | ||
The path to the private key to be used as the ssh identity file. When ``auth_type`` is | ||
``publickey`` this is a required config. The key has to be readable by user ``apache``. | ||
|
||
``ssh_password`` | ||
The password to be used for ``ssh_user`` on the remote server. ``ssh_password`` is required when | ||
``auth_type`` is 'password'. | ||
|
||
``host`` | ||
The hostname of the remote server. | ||
|
||
``root`` | ||
The absolute path to the remote root directory where all the data (content and published content) | ||
lives. This is the remote equivalent to ``/var/lib/pulp``. The repo id is appended to the | ||
``root`` path to determine the location of published repository. | ||
|
||
Optional configuration | ||
---------------------- | ||
|
||
``content_units_only`` | ||
If true, the distributor will publish content units only (e.g. ``/var/lib/pulp/content``). The | ||
symlinks of a published repository will not be rsynced. | ||
|
||
``delete`` | ||
If true, ``--delete`` is appended to the rsync command for symlinks and repodata so that any old | ||
files no longer present in the local published directory are removed from the remote server. | ||
|
||
``remote_units_path`` | ||
The relative path from the ``root`` where unit files will live. Defaults to ``content/units``. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
105 changes: 105 additions & 0 deletions
105
plugins/pulp_rpm/plugins/distributors/iso_rsync_distributor/distributor.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
import logging | ||
from gettext import gettext as _ | ||
|
||
from pulp.common.config import read_json_config | ||
from pulp.plugins.distributor import Distributor | ||
from pulp.plugins.rsync import configuration | ||
|
||
from pulp_rpm.common.ids import TYPE_ID_ISO | ||
from pulp_rpm.plugins.distributors.iso_rsync_distributor import publish | ||
|
||
_LOG = logging.getLogger(__name__) | ||
|
||
TYPE_ID_DISTRIBUTOR_ISO_RSYNC = 'iso_rsync_distributor' | ||
CONF_FILE_PATH = 'server/plugins.conf.d/%s.json' % TYPE_ID_DISTRIBUTOR_ISO_RSYNC | ||
|
||
DISTRIBUTOR_DISPLAY_NAME = 'ISO Rsync Distributor' | ||
|
||
|
||
def entry_point(): | ||
config = read_json_config(CONF_FILE_PATH) | ||
return ISORsyncDistributor, config | ||
|
||
|
||
class ISORsyncDistributor(Distributor): | ||
""" | ||
Distributor class for publishing RPM repo to remote server. | ||
:ivar canceled: if true, the task has been canceled | ||
:ivar _publisher: instance of RPMRsyncPublisher | ||
""" | ||
|
||
def __init__(self): | ||
super(ISORsyncDistributor, self).__init__() | ||
self.canceled = False | ||
self._publisher = None | ||
|
||
@classmethod | ||
def metadata(cls): | ||
""" | ||
Used by Pulp to classify the capabilities of this distributor. | ||
:return: description of the distributor's capabilities | ||
:rtype: dict | ||
""" | ||
return {'id': TYPE_ID_DISTRIBUTOR_ISO_RSYNC, | ||
'display_name': DISTRIBUTOR_DISPLAY_NAME, | ||
'types': [TYPE_ID_ISO]} | ||
|
||
# -- repo lifecycle methods ------------------------------------------------ | ||
|
||
def validate_config(self, repo, config, config_conduit): | ||
""" | ||
Allows the distributor to check the contents of a potential configuration | ||
for the given repository. This call is made both for the addition of | ||
this distributor to a new repository as well as updating the configuration | ||
for this distributor on a previously configured repository. | ||
:param repo: metadata describing the repository to which the | ||
configuration applies | ||
:type repo: pulp.plugins.model.Repository | ||
:param config: plugin configuration instance; the proposed repo | ||
configuration is found within | ||
:type config: pulp.plugins.config.PluginCallConfiguration | ||
:param config_conduit: Configuration Conduit; | ||
:type config_conduit: pulp.plugins.conduits.repo_config.RepoConfigConduit | ||
:return: tuple of (bool, str) to describe the result | ||
:rtype: tuple | ||
""" | ||
_LOG.debug(_('Validating iso repository configuration: %(repoid)s') % {'repoid': repo.id}) | ||
return configuration.validate_config(repo, config, config_conduit) | ||
|
||
# -- actions --------------------------------------------------------------- | ||
|
||
def publish_repo(self, repo, publish_conduit, config): | ||
""" | ||
Publishes the given repository. | ||
:param repo: metadata describing the repository | ||
:type repo: pulp.plugins.model.Repository | ||
:param publish_conduit: provides access to relevant Pulp functionality | ||
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit | ||
:param config: plugin configuration | ||
:type config: pulp.plugins.config.PluginConfiguration | ||
:return: report describing the publish run | ||
:rtype: pulp.plugins.model.PublishReport | ||
""" | ||
_LOG.debug(_('Publishing ISO repository: %(repoid)s') % {'repoid': repo.id}) | ||
self._publisher = publish.ISORsyncPublisher(repo, publish_conduit, config, | ||
TYPE_ID_DISTRIBUTOR_ISO_RSYNC) | ||
return self._publisher.publish() | ||
|
||
def cancel_publish_repo(self): | ||
""" | ||
Call cancellation control hook. | ||
""" | ||
_LOG.debug(_('Canceling publishing repo to remote server')) | ||
self.canceled = True | ||
if self._publisher is not None: | ||
self._publisher.cancel() |
105 changes: 105 additions & 0 deletions
105
plugins/pulp_rpm/plugins/distributors/iso_rsync_distributor/publish.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
import os | ||
|
||
from pulp.plugins.rsync.publish import Publisher, RSyncPublishStep | ||
from pulp.plugins.util.publish_step import RSyncFastForwardUnitPublishStep | ||
from pulp.server.db.model import Distributor | ||
from pulp.server.exceptions import PulpCodedException | ||
|
||
from pulp_rpm.common import constants | ||
from pulp_rpm.plugins.db import models | ||
from pulp_rpm.plugins import error_codes | ||
|
||
|
||
class ISORsyncPublisher(Publisher): | ||
|
||
REPO_CONTENT_MODELS = [models.ISO] | ||
|
||
UNIT_FIELDS = ["_storage_path", "name"] | ||
|
||
def _get_predistributor(self): | ||
""" | ||
Returns the distributor that is configured as predistributor. | ||
""" | ||
predistributor = self.get_config().flatten().get("predistributor_id", None) | ||
if predistributor: | ||
return Distributor.objects.get_or_404(repo_id=self.repo.id, | ||
distributor_id=predistributor) | ||
else: | ||
raise PulpCodedException(error_code=error_codes.RPM1011) | ||
|
||
def _get_root_publish_dir(self): | ||
""" | ||
Returns the publish directory path for the predistributor. | ||
:return: absolute path to the master publish directory of predistributor | ||
:rtype: str | ||
""" | ||
if self.predistributor["config"].get("http", False): | ||
return constants.ISO_HTTP_DIR | ||
else: | ||
return constants.ISO_HTTPS_DIR | ||
|
||
def get_master_directory(self): | ||
""" | ||
Returns path to master directory of the predistributor. | ||
:return: path to 'master' publish directory | ||
:rtype: str | ||
""" | ||
return os.path.join(self._get_root_publish_dir(), self.repo.id) | ||
|
||
def _add_necesary_steps(self, date_filter=None, config=None): | ||
""" | ||
This method adds all the steps that are needed to accomplish an ISO rsync publish. This | ||
includes: | ||
Unit Query Step - selects units associated with the repo based on the date_filter and | ||
creates relative symlinks | ||
Rsync Step (content units) - rsyncs units discovered in previous step to the remote server | ||
Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server | ||
Rsync Step (PULP_MANIFEST) - rsyncs PULP_MANIFEST to remote server | ||
:param date_filter: Q object with start and/or end dates, or None if start and end dates | ||
are not provided | ||
:type date_filter: mongoengine.Q or types.NoneType | ||
:param config: Pulp configuration for the distributor | ||
:type config: pulp.plugins.config.PluginCallConfiguration | ||
""" | ||
remote_repo_path = self.repo.id | ||
|
||
# Find all the units associated with repo before last publish with predistributor | ||
gen_step = RSyncFastForwardUnitPublishStep("Unit query step (ISO)", | ||
ISORsyncPublisher.REPO_CONTENT_MODELS, | ||
repo=self.repo, | ||
remote_repo_path=remote_repo_path, | ||
published_unit_path=[], | ||
unit_fields=ISORsyncPublisher.UNIT_FIELDS) | ||
self.add_child(gen_step) | ||
|
||
dest_content_units_dir = self.get_units_directory_dest_path() | ||
src_content_units_dir = self.get_units_src_path() | ||
|
||
# Rsync content units | ||
self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list, | ||
src_content_units_dir, dest_content_units_dir, | ||
config=config, exclude=[])) | ||
|
||
# Stop here if distributor is only supposed to publish actual content | ||
if self.get_config().flatten().get("content_units_only"): | ||
return | ||
|
||
# Rsync symlinks to the remote server | ||
self.add_child(RSyncPublishStep("Rsync step (symlinks)", | ||
self.symlink_list, self.symlink_src, | ||
remote_repo_path, | ||
config=config, links=True, exclude=["PULP_MANIFEST"], | ||
delete=self.config.get("delete"))) | ||
|
||
predistributor_master_dir = self.get_master_directory() | ||
|
||
# Rsync PULP_MANIFEST | ||
self.add_child(RSyncPublishStep("Rsync step (PULP_MANIFEST)", | ||
['PULP_MANIFEST'], predistributor_master_dir, | ||
remote_repo_path, | ||
config=config)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters