diff --git a/pubtools/pulplib/__init__.py b/pubtools/pulplib/__init__.py index 2a95db6c..3399acb2 100644 --- a/pubtools/pulplib/__init__.py +++ b/pubtools/pulplib/__init__.py @@ -16,6 +16,10 @@ ModulemdDefaultsUnit, Distributor, PublishOptions, + FileSyncOptions, + ContainerSyncOptions, + YumSyncOptions, + SyncOptions, Task, MaintenanceReport, MaintenanceEntry, diff --git a/pubtools/pulplib/_impl/client/client.py b/pubtools/pulplib/_impl/client/client.py index 65c52558..0caef8e2 100644 --- a/pubtools/pulplib/_impl/client/client.py +++ b/pubtools/pulplib/_impl/client/client.py @@ -229,7 +229,7 @@ def _search( search_type="search", search_options=None, criteria=None, - ): + ): # pylint:disable = too-many-arguments url = os.path.join( self._url, "pulp/api/v2/%s/%s/" % (resource_type, search_type) ) @@ -565,3 +565,18 @@ def map_404_to_none(exception): ) return f_map(response, error_fn=map_404_to_none) + + def _do_sync(self, repo_id, sync_options): + if not sync_options["feed"]: + raise ValueError("Cannot sync with empty feed: '%s'" % sync_options["feed"]) + + url = os.path.join( + self._url, "pulp/api/v2/repositories/%s/actions/sync/" % repo_id + ) + + body = {"override_config": sync_options} + + LOG.debug("Syncing repository %s with feed %s", repo_id, sync_options["feed"]) + return self._task_executor.submit( + self._do_request, method="POST", url=url, json=body + ) diff --git a/pubtools/pulplib/_impl/fake/client.py b/pubtools/pulplib/_impl/fake/client.py index 0960be32..21865b27 100644 --- a/pubtools/pulplib/_impl/fake/client.py +++ b/pubtools/pulplib/_impl/fake/client.py @@ -29,9 +29,10 @@ Publish = namedtuple("Publish", ["repository", "tasks"]) Upload = namedtuple("Upload", ["repository", "tasks", "name", "sha256"]) +Sync = namedtuple("Sync", ["repository", "tasks", "sync_config"]) -class FakeClient(object): +class FakeClient(object): # pylint:disable = too-many-instance-attributes # Client implementation holding data in memory rather than # using a remote Pulp server. # @@ -62,6 +63,7 @@ def __init__(self): self._repo_units = {} self._publish_history = [] self._upload_history = [] + self._sync_history = [] self._maintenance_report = None self._type_ids = self._DEFAULT_TYPE_IDS[:] self._lock = threading.RLock() @@ -356,6 +358,18 @@ def _attach_repo(self, repo): repo._set_client(self) return repo + def _do_sync(self, repo_id, sync_config): # pylint:disable = unused-argument + repo_f = self.get_repository(repo_id) + if repo_f.exception(): + # Repo can't be found, let that exception propagate + return repo_f + + task = Task(id=self._next_task_id(), completed=True, succeeded=True) + + self._sync_history.append(Sync(repo_f.result(), [task], sync_config)) + + return f_return([task]) + def _next_task_id(self): with self._lock: next_raw_id = self._uuidgen.randint(0, 2 ** 128) diff --git a/pubtools/pulplib/_impl/fake/controller.py b/pubtools/pulplib/_impl/fake/controller.py index ec8921ed..3b85b2d2 100644 --- a/pubtools/pulplib/_impl/fake/controller.py +++ b/pubtools/pulplib/_impl/fake/controller.py @@ -112,6 +112,23 @@ def publish_history(self): """ return self.client._publish_history[:] + @property + def sync_history(self): + """A list of repository syncs triggered via this client. + + Each element of this list is a named tuple with the following attributes, + in order: + + ``repository``: + :class:`~pubtools.pulplib.Repository` for which sync was triggered + ``tasks``: + list of :class:`~pubtools.pulplib.Task` generated as a result + of this sync + ``sync_config``: + :class:`~pubtools.pulplib.SyncConfig` (of the appropriate subclass) used for this sync + """ + return self.client._sync_history[:] + @property def upload_history(self): """A list of upload tasks triggered via this client. diff --git a/pubtools/pulplib/_impl/fake/match.py b/pubtools/pulplib/_impl/fake/match.py index 5b4a305f..09515698 100644 --- a/pubtools/pulplib/_impl/fake/match.py +++ b/pubtools/pulplib/_impl/fake/match.py @@ -29,7 +29,7 @@ def wrap(func): return wrap -def match_object(*args, **kwargs): +def match_object(*args, **kwargs): # pylint:disable=inconsistent-return-statements dispatch = args[0] for (klass, func) in CLASS_MATCHERS: if isinstance(dispatch, klass): diff --git a/pubtools/pulplib/_impl/model/__init__.py b/pubtools/pulplib/_impl/model/__init__.py index 8eeb7dc5..9a35e46b 100644 --- a/pubtools/pulplib/_impl/model/__init__.py +++ b/pubtools/pulplib/_impl/model/__init__.py @@ -9,6 +9,10 @@ FileRepository, ContainerImageRepository, PublishOptions, + SyncOptions, + FileSyncOptions, + ContainerSyncOptions, + YumSyncOptions, ) from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit, ModulemdDefaultsUnit from .task import Task diff --git a/pubtools/pulplib/_impl/model/repository/__init__.py b/pubtools/pulplib/_impl/model/repository/__init__.py index 1bdeaf17..7fa39a87 100644 --- a/pubtools/pulplib/_impl/model/repository/__init__.py +++ b/pubtools/pulplib/_impl/model/repository/__init__.py @@ -1,4 +1,4 @@ -from .base import Repository, PublishOptions -from .container import ContainerImageRepository -from .yum import YumRepository -from .file import FileRepository +from .base import Repository, PublishOptions, SyncOptions +from .container import ContainerImageRepository, ContainerSyncOptions +from .yum import YumRepository, YumSyncOptions +from .file import FileRepository, FileSyncOptions diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 27b35a64..054ce6e8 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -1,14 +1,10 @@ import datetime import logging -from attr import validators +from attr import validators, asdict from more_executors.futures import f_proxy -from ..common import ( - PulpObject, - Deletable, - DetachedException, -) +from ..common import PulpObject, Deletable, DetachedException from ..attr import pulp_attrib from ..distributor import Distributor from ..frozenlist import FrozenList @@ -62,6 +58,55 @@ class PublishOptions(object): """ +@attr.s(kw_only=True, frozen=True) +class SyncOptions(object): + """Options controlling a repository + :meth:`~pubtools.pulplib.Repository.sync`. + """ + + feed = pulp_attrib(type=str) + """URL where the repository's content will be synchronized from. + """ + + ssl_validation = pulp_attrib(default=None, type=bool) + """Indicates if the server's SSL certificate is verified against the CA certificate uploaded. + """ + + ssl_ca_cert = pulp_attrib(default=None, type=str) + """CA certificate string used to validate the feed source's SSL certificate + """ + + ssl_client_cert = pulp_attrib(default=None, type=str) + """Certificate used as the client certificate when synchronizing the repository + """ + + ssl_client_key = pulp_attrib(default=None, type=str) + """Private key to the certificate specified in ssl_client_cert + """ + + max_speed = pulp_attrib(default=None, type=int) + """The maximum download speed in bytes/sec for a task (such as a sync). + + Default is None + """ + + proxy_host = pulp_attrib(default=None, type=str) + """A string representing the URL of the proxy server that should be used when synchronizing + """ + + proxy_port = pulp_attrib(default=None, type=int) + """An integer representing the port that should be used when connecting to proxy_host. + """ + + proxy_username = pulp_attrib(default=None, type=str) + """A string representing the username that should be used to authenticate with the proxy server + """ + + proxy_password = pulp_attrib(default=None, type=str) + """A string representing the password that should be used to authenticate with the proxy server + """ + + @attr.s(kw_only=True, frozen=True) class Repository(PulpObject, Deletable): """Represents a Pulp repository.""" @@ -168,7 +213,7 @@ def _check_repo_id(self, _, value): for distributor in value: if not distributor.repo_id: return - elif distributor.repo_id == self.id: + if distributor.repo_id == self.id: return raise ValueError( "repo_id doesn't match for %s. repo_id: %s, distributor.repo_id: %s" @@ -343,6 +388,34 @@ def publish(self, options=PublishOptions()): return f_proxy(self._client._publish_repository(self, to_publish)) + def sync(self, options=SyncOptions(feed="")): + """Sync repository with feed + + Args: + options (SyncOptions) + Options used to customize the behavior of sync process. + If omitted, the Pulp server's defaults apply. + + Returns: + Future[list[:class:`~pubtools.pulplib.Task`]] + A future which is resolved when sync succeeds. + + The future contains a list of zero or more tasks triggered and awaited + during the sync operation. + + Raises: + DetachedException + If this instance is not attached to a Pulp client. + """ + if not self._client: + raise DetachedException() + + return f_proxy( + self._client._do_sync( + self.id, asdict(options, filter=lambda name, val: val is not None) + ) + ) + def remove_content(self, **kwargs): """Remove all content of requested types from this repository. diff --git a/pubtools/pulplib/_impl/model/repository/container.py b/pubtools/pulplib/_impl/model/repository/container.py index 5005155f..6b1c71b8 100644 --- a/pubtools/pulplib/_impl/model/repository/container.py +++ b/pubtools/pulplib/_impl/model/repository/container.py @@ -1,8 +1,24 @@ -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..attr import pulp_attrib from ... import compat_attr as attr +@attr.s(kw_only=True, frozen=True) +class ContainerSyncOptions(SyncOptions): + """Options controlling a container repository + :meth:`~pubtools.pulplib.ContainerImageRepository.sync`. + """ + + upstream_name = pulp_attrib(default=None, type=str) + """The name of the repository to import from the upstream repository. + For example, if syncing from repository `quay.io/fedora/fedora`, upstream_name should be set to `fedora/fedora`. + """ + + tags = pulp_attrib(default=None, type=list) + """List of tags to include on sync. + """ + + @repo_type("docker-repo") @attr.s(kw_only=True, frozen=True) class ContainerImageRepository(Repository): diff --git a/pubtools/pulplib/_impl/model/repository/file.py b/pubtools/pulplib/_impl/model/repository/file.py index 0606efba..5e5090f6 100644 --- a/pubtools/pulplib/_impl/model/repository/file.py +++ b/pubtools/pulplib/_impl/model/repository/file.py @@ -4,7 +4,7 @@ from attr import validators from more_executors.futures import f_flat_map, f_map, f_proxy -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..frozenlist import FrozenList from ..attr import pulp_attrib from ..common import DetachedException @@ -14,6 +14,17 @@ LOG = logging.getLogger("pubtools.pulplib") +@attr.s(kw_only=True, frozen=True) +class FileSyncOptions(SyncOptions): + """Options controlling a file repository + :meth:`~pubtools.pulplib.FileRepository.sync`. + """ + + remove_missing = pulp_attrib(default=False, type=bool) + """If true, as the repository is synchronized, old files will be removed. + """ + + @repo_type("iso-repo") @attr.s(kw_only=True, frozen=True) class FileRepository(Repository): diff --git a/pubtools/pulplib/_impl/model/repository/yum.py b/pubtools/pulplib/_impl/model/repository/yum.py index 6057ea65..450aa558 100644 --- a/pubtools/pulplib/_impl/model/repository/yum.py +++ b/pubtools/pulplib/_impl/model/repository/yum.py @@ -1,9 +1,66 @@ -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..frozenlist import FrozenList from ..attr import pulp_attrib from ... import compat_attr as attr +@attr.s(kw_only=True, frozen=True) +class YumSyncOptions(SyncOptions): + """Options controlling a container repository + :meth:`~pubtools.pulplib.YumRepository.sync`. + """ + + query_auth_token = pulp_attrib(default=None, type=str) + """An authorization token that will be added to every request made to the feed URL's server + """ + + max_downloads = pulp_attrib(default=None, type=int) + """Number of threads used when synchronizing the repository. + """ + + remove_missing = pulp_attrib(default=None, type=bool) + """If true, as the repository is synchronized, old rpms will be removed. + """ + + retain_old_count = pulp_attrib(default=None, type=int) + """Count indicating how many old rpm versions to retain. + """ + + skip = pulp_attrib(default=None, type=list) + """List of content types to be skipped during the repository synchronization + """ + + checksum_type = pulp_attrib(default=None, type=str) + """checksum type to use for metadata generation. + + Defaults to source checksum type of sha256 + """ + + num_retries = pulp_attrib(default=None, type=int) + """Number of times to retry before declaring an error during repository synchronization + + Default is to 2. + """ + + download_policy = pulp_attrib(default=None, type=str) + """Set the download policy for a repository. + + Supported options are immediate,on_demand,background + """ + + force_full = pulp_attrib(default=None, type=bool) + """Boolean flag. If true, full re-sync is triggered. + """ + + require_signature = pulp_attrib(default=None, type=bool) + """Requires that imported packages like RPM/DRPM/SRPM should be signed + """ + + allowed_keys = pulp_attrib(default=None, type=list) + """List of allowed signature key IDs that imported packages can be signed with + """ + + @repo_type("rpm-repo") @attr.s(kw_only=True, frozen=True) class YumRepository(Repository): diff --git a/setup.py b/setup.py index 5feb4896..b2fde7fd 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def get_requirements(): setup( name="pubtools-pulplib", - version="2.4.0", + version="2.5.0", packages=find_packages(exclude=["tests"]), package_data={"pubtools.pulplib._impl.schema": ["*.yaml"]}, url="https://github.com/release-engineering/pubtools-pulplib", diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py new file mode 100644 index 00000000..c42f6a0f --- /dev/null +++ b/tests/fake/test_fake_sync.py @@ -0,0 +1,54 @@ +from pubtools.pulplib import ( + FakeController, + YumRepository, + YumSyncOptions, + PulpException, +) + + +def test_can_sync(): + """repo.sync() succeeds with fake client and populates sync_history.""" + controller = FakeController() + + controller.insert_repository(YumRepository(id="repo1")) + controller.insert_repository(YumRepository(id="repo2")) + + client = controller.client + repo1 = client.get_repository("repo1") + + # Call to sync should succeed + sync_f = repo1.sync(YumSyncOptions(feed="mock://feed/")) + + # The future should resolve successfully + tasks = sync_f.result() + + # It should have returned at least one successful task. + assert tasks + for task in tasks: + assert task.succeeded + + # The change should be reflected in the controller's sync history + history = controller.sync_history + + assert len(history) == 1 + assert history[0].repository.id == "repo1" + assert history[0].tasks == tasks + + +def test_sync_absent_raises(): + """repo.sync() of a nonexistent repo raises.""" + controller = FakeController() + + controller.insert_repository(YumRepository(id="repo1")) + + client = controller.client + repo_copy1 = client.get_repository("repo1") + repo_copy2 = client.get_repository("repo1") + + # If I delete the repo through one handle... + assert repo_copy1.delete().result() + + # ...then sync through the other handle becomes impossible + exception = repo_copy2.sync(YumSyncOptions(feed="mock://feed/")).exception() + assert isinstance(exception, PulpException) + assert "repo1 not found" in str(exception) diff --git a/tests/repository/test_sync.py b/tests/repository/test_sync.py new file mode 100644 index 00000000..691a82e7 --- /dev/null +++ b/tests/repository/test_sync.py @@ -0,0 +1,75 @@ +import logging +import pytest + +from pubtools.pulplib import ( + Repository, + YumRepository, + Task, + Distributor, + DetachedException, + YumSyncOptions, + TaskFailedException, +) + + +@pytest.fixture +def fixture_sync_async_response(requests_mocker): + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/sync/", + [{"json": {"spawned_tasks": [{"task_id": "task1"}]}}], + ) + + +@pytest.fixture +def fixture_search_task_response(requests_mocker): + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/tasks/search/", + [{"json": [{"task_id": "task1", "state": "finished"}]}], + ) + + +def test_detached(): + """sync raises if called on a detached repo""" + with pytest.raises(DetachedException): + Repository(id="some-repo").sync() + + +def test_sync_no_feed( + fast_poller, + client, + requests_mocker, + fixture_sync_async_response, + fixture_search_task_response, +): + """Test sync fail as no feed is provided.""" + repo = YumRepository(id="some-repo") + repo.__dict__["_client"] = client + + # empty options should fail as feed is required to be non-empty + try: + repo.sync().result() + assert "Exception should have been raised" + except ValueError: + pass + + +def test_sync_with_options( + requests_mocker, client, fixture_sync_async_response, fixture_search_task_response +): + """Test sync passes, test whether sync options are passed to override config.""" + repo = YumRepository(id="some-repo") + repo.__dict__["_client"] = client + + options = YumSyncOptions(ssl_validation=False, feed="mock://example.com/") + + # It should have succeeded, with the tasks as retrieved from Pulp + assert repo.sync(options).result() == [ + Task(id="task1", succeeded=True, completed=True) + ] + + req = requests_mocker.request_history + + assert req[0].json()["override_config"] == { + "ssl_validation": False, + "feed": "mock://example.com/", + }