Support pulp sync repository operation

pubtools/pulplib/__init__.py

@@ -16,6 +16,10 @@
     ModulemdDefaultsUnit,
     Distributor,
     PublishOptions,
+    FileSyncOptions,
+    ContainerSyncOptions,
+    YumSyncOptions,
+    SyncOptions,
     Task,
     MaintenanceReport,
     MaintenanceEntry,

pubtools/pulplib/_impl/client/client.py

@@ -229,7 +229,7 @@ def _search(
         search_type="search",
         search_options=None,
         criteria=None,
-    ):
+    ):  # pylint:disable = too-many-arguments
         url = os.path.join(
             self._url, "pulp/api/v2/%s/%s/" % (resource_type, search_type)
         )
@@ -565,3 +565,18 @@ def map_404_to_none(exception):
         )
 
         return f_map(response, error_fn=map_404_to_none)
+
+    def _do_sync(self, repo_id, sync_options):
+        if not sync_options["feed"]:
+            raise ValueError("Cannot sync with empty feed: '%s'" % sync_options["feed"])
+
+        url = os.path.join(
+            self._url, "pulp/api/v2/repositories/%s/actions/sync/" % repo_id
+        )
+
+        body = {"override_config": sync_options}
+
+        LOG.debug("Syncing repository %s with feed %s", repo_id, sync_options["feed"])
+        return self._task_executor.submit(
+            self._do_request, method="POST", url=url, json=body
+        )

pubtools/pulplib/_impl/fake/client.py

@@ -29,9 +29,10 @@
 
 Publish = namedtuple("Publish", ["repository", "tasks"])
 Upload = namedtuple("Upload", ["repository", "tasks", "name", "sha256"])
+Sync = namedtuple("Sync", ["repository", "tasks", "sync_config"])
 
 
-class FakeClient(object):
+class FakeClient(object):  # pylint:disable = too-many-instance-attributes
     # Client implementation holding data in memory rather than
     # using a remote Pulp server.
     #
@@ -62,6 +63,7 @@ def __init__(self):
         self._repo_units = {}
         self._publish_history = []
         self._upload_history = []
+        self._sync_history = []
         self._maintenance_report = None
         self._type_ids = self._DEFAULT_TYPE_IDS[:]
         self._lock = threading.RLock()
@@ -356,6 +358,18 @@ def _attach_repo(self, repo):
         repo._set_client(self)
         return repo
 
+    def _do_sync(self, repo_id, sync_config):  # pylint:disable = unused-argument
+        repo_f = self.get_repository(repo_id)
+        if repo_f.exception():
+            # Repo can't be found, let that exception propagate
+            return repo_f
+
+        task = Task(id=self._next_task_id(), completed=True, succeeded=True)
+
+        self._sync_history.append(Sync(repo_f.result(), [task], sync_config))
+
+        return f_return([task])
+
     def _next_task_id(self):
         with self._lock:
             next_raw_id = self._uuidgen.randint(0, 2 ** 128)

pubtools/pulplib/_impl/fake/controller.py

@@ -112,6 +112,23 @@ def publish_history(self):
         """
         return self.client._publish_history[:]
 
+    @property
+    def sync_history(self):
+        """A list of repository syncs triggered via this client.
+
+        Each element of this list is a named tuple with the following attributes,
+        in order:
+
+            ``repository``:
+                :class:`~pubtools.pulplib.Repository` for which sync was triggered
+            ``tasks``:
+                list of :class:`~pubtools.pulplib.Task` generated as a result
+                of this sync
+            ``sync_config``:
+                :class:`~pubtools.pulplib.SyncConfig` (of the appropriate subclass) used for this sync
+        """
+        return self.client._sync_history[:]
+
     @property
     def upload_history(self):
         """A list of upload tasks triggered via this client.

pubtools/pulplib/_impl/fake/match.py

@@ -29,7 +29,7 @@ def wrap(func):
     return wrap
 
 
-def match_object(*args, **kwargs):
+def match_object(*args, **kwargs):  # pylint:disable=inconsistent-return-statements
     dispatch = args[0]
     for (klass, func) in CLASS_MATCHERS:
         if isinstance(dispatch, klass):

pubtools/pulplib/_impl/model/__init__.py

@@ -9,6 +9,10 @@
     FileRepository,
     ContainerImageRepository,
     PublishOptions,
+    SyncOptions,
+    FileSyncOptions,
+    ContainerSyncOptions,
+    YumSyncOptions,
 )
 from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit, ModulemdDefaultsUnit
 from .task import Task

pubtools/pulplib/_impl/model/repository/__init__.py

@@ -1,4 +1,4 @@
-from .base import Repository, PublishOptions
-from .container import ContainerImageRepository
-from .yum import YumRepository
-from .file import FileRepository
+from .base import Repository, PublishOptions, SyncOptions
+from .container import ContainerImageRepository, ContainerSyncOptions
+from .yum import YumRepository, YumSyncOptions
+from .file import FileRepository, FileSyncOptions

pubtools/pulplib/_impl/model/repository/base.py

@@ -1,14 +1,10 @@
 import datetime
 import logging
 
-from attr import validators
+from attr import validators, asdict
 from more_executors.futures import f_proxy
 
-from ..common import (
-    PulpObject,
-    Deletable,
-    DetachedException,
-)
+from ..common import PulpObject, Deletable, DetachedException
 from ..attr import pulp_attrib
 from ..distributor import Distributor
 from ..frozenlist import FrozenList
@@ -62,6 +58,55 @@ class PublishOptions(object):
     """
 
 
+@attr.s(kw_only=True, frozen=True)
+class SyncOptions(object):
+    """Options controlling a repository
+    :meth:`~pubtools.pulplib.Repository.sync`.
+    """
+
+    feed = pulp_attrib(type=str)
+    """URL where the repository's content will be synchronized from.
+    """
+
+    ssl_validation = pulp_attrib(default=None, type=bool)
+    """Indicates if the server's SSL certificate is verified against the CA certificate uploaded.
+    """
+
+    ssl_ca_cert = pulp_attrib(default=None, type=str)
+    """CA certificate string used to validate the feed source's SSL certificate
+    """
+
+    ssl_client_cert = pulp_attrib(default=None, type=str)
+    """Certificate used as the client certificate when synchronizing the repository
+    """
+
+    ssl_client_key = pulp_attrib(default=None, type=str)
+    """Private key to the certificate specified in ssl_client_cert
+    """
+
+    max_speed = pulp_attrib(default=None, type=int)
+    """The maximum download speed in bytes/sec for a task (such as a sync).
+
+    Default is None
+    """
+
+    proxy_host = pulp_attrib(default=None, type=str)
+    """A string representing the URL of the proxy server that should be used when synchronizing
+    """
+
+    proxy_port = pulp_attrib(default=None, type=int)
+    """An integer representing the port that should be used when connecting to proxy_host.
+    """
+
+    proxy_username = pulp_attrib(default=None, type=str)
+    """A string representing the username that should be used to authenticate with the proxy server
+    """
+
+    proxy_password = pulp_attrib(default=None, type=str)
+    """A string representing the password that should be used to authenticate with the proxy server
+    """
+
+
 @attr.s(kw_only=True, frozen=True)
 class Repository(PulpObject, Deletable):
     """Represents a Pulp repository."""
@@ -168,7 +213,7 @@ def _check_repo_id(self, _, value):
         for distributor in value:
             if not distributor.repo_id:
                 return
-            elif distributor.repo_id == self.id:
+            if distributor.repo_id == self.id:
                 return
             raise ValueError(
                 "repo_id doesn't match for %s. repo_id: %s, distributor.repo_id: %s"
@@ -343,6 +388,34 @@ def publish(self, options=PublishOptions()):
 
         return f_proxy(self._client._publish_repository(self, to_publish))
 
+    def sync(self, options=SyncOptions(feed="")):
+        """Sync repository with feed
+
+        Args:
+            options (SyncOptions)
+                Options used to customize the behavior of sync process.
+                If omitted, the Pulp server's defaults apply.
+
+        Returns:
+            Future[list[:class:`~pubtools.pulplib.Task`]]
+                A future which is resolved when sync succeeds.
+
+                The future contains a list of zero or more tasks triggered and awaited
+                during the sync operation.
+
+        Raises:
+            DetachedException
+                If this instance is not attached to a Pulp client.
+        """
+        if not self._client:
+            raise DetachedException()
+
+        return f_proxy(
+            self._client._do_sync(
+                self.id, asdict(options, filter=lambda name, val: val is not None)
+            )
+        )
+
     def remove_content(self, **kwargs):
         """Remove all content of requested types from this repository.
 

pubtools/pulplib/_impl/model/repository/container.py

@@ -1,8 +1,24 @@
-from .base import Repository, repo_type
+from .base import Repository, SyncOptions, repo_type
 from ..attr import pulp_attrib
 from ... import compat_attr as attr
 
 
+@attr.s(kw_only=True, frozen=True)
+class ContainerSyncOptions(SyncOptions):
+    """Options controlling a container repository
+    :meth:`~pubtools.pulplib.ContainerImageRepository.sync`.
+    """
+
+    upstream_name = pulp_attrib(default=None, type=str)
+    """The name of the repository to import from the upstream repository.
+    For example, if syncing from repository `quay.io/fedora/fedora`, upstream_name should be set to `fedora/fedora`.
+    """
+
+    tags = pulp_attrib(default=None, type=list)
+    """List of tags to include on sync.
+    """
+
+
 @repo_type("docker-repo")
 @attr.s(kw_only=True, frozen=True)
 class ContainerImageRepository(Repository):

pubtools/pulplib/_impl/model/repository/file.py

@@ -4,7 +4,7 @@
 from attr import validators
 from more_executors.futures import f_flat_map, f_map, f_proxy
 
-from .base import Repository, repo_type
+from .base import Repository, SyncOptions, repo_type
 from ..frozenlist import FrozenList
 from ..attr import pulp_attrib
 from ..common import DetachedException
@@ -14,6 +14,17 @@
 LOG = logging.getLogger("pubtools.pulplib")
 
 
+@attr.s(kw_only=True, frozen=True)
+class FileSyncOptions(SyncOptions):
+    """Options controlling a file repository
+    :meth:`~pubtools.pulplib.FileRepository.sync`.
+    """
+
+    remove_missing = pulp_attrib(default=False, type=bool)
+    """If true, as the repository is synchronized, old files will be removed.
+    """
+
+
 @repo_type("iso-repo")
 @attr.s(kw_only=True, frozen=True)
 class FileRepository(Repository):

pubtools/pulplib/_impl/model/repository/yum.py

@@ -1,9 +1,66 @@
-from .base import Repository, repo_type
+from .base import Repository, SyncOptions, repo_type
 from ..frozenlist import FrozenList
 from ..attr import pulp_attrib
 from ... import compat_attr as attr
 
 
+@attr.s(kw_only=True, frozen=True)
+class YumSyncOptions(SyncOptions):
+    """Options controlling a container repository
+    :meth:`~pubtools.pulplib.YumRepository.sync`.
+    """
+
+    query_auth_token = pulp_attrib(default=None, type=str)
+    """An authorization token that will be added to every request made to the feed URL's server
+    """
+
+    max_downloads = pulp_attrib(default=None, type=int)
+    """Number of threads used when synchronizing the repository.
+    """
+
+    remove_missing = pulp_attrib(default=None, type=bool)
+    """If true, as the repository is synchronized, old rpms will be removed.
+    """
+
+    retain_old_count = pulp_attrib(default=None, type=int)
+    """Count indicating how many old rpm versions to retain.
+    """
+
+    skip = pulp_attrib(default=None, type=list)
+    """List of content types to be skipped during the repository synchronization
+    """
+
+    checksum_type = pulp_attrib(default=None, type=str)
+    """checksum type to use for metadata generation.
+
+    Defaults to source checksum type of sha256
+    """
+
+    num_retries = pulp_attrib(default=None, type=int)
+    """Number of times to retry before declaring an error during repository synchronization
+
+    Default is to 2.
+    """
+
+    download_policy = pulp_attrib(default=None, type=str)
+    """Set the download policy for a repository.
+
+    Supported options are immediate,on_demand,background
+    """
+
+    force_full = pulp_attrib(default=None, type=bool)
+    """Boolean flag. If true, full re-sync is triggered.
+    """
+
+    require_signature = pulp_attrib(default=None, type=bool)
+    """Requires that imported packages like RPM/DRPM/SRPM should be signed
+    """
+
+    allowed_keys = pulp_attrib(default=None, type=list)
+    """List of allowed signature key IDs that imported packages can be signed with
+    """
+
+
 @repo_type("rpm-repo")
 @attr.s(kw_only=True, frozen=True)
 class YumRepository(Repository):

setup.py

@@ -21,7 +21,7 @@ def get_requirements():
 
 setup(
     name="pubtools-pulplib",
-    version="2.4.0",
+    version="2.5.0",
     packages=find_packages(exclude=["tests"]),
     package_data={"pubtools.pulplib._impl.schema": ["*.yaml"]},
     url="https://github.com/release-engineering/pubtools-pulplib",