From 20ca4144664f5c2ce77630afca33dcbcd20915b9 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Wed, 12 Feb 2020 14:56:57 +0100 Subject: [PATCH 01/19] Support pulp sync repository operation Repository.sync() operation is now supported for Yum,File and ContainerImage Repository --- pubtools/pulplib/__init__.py | 1 + pubtools/pulplib/_impl/client/client.py | 17 +++- pubtools/pulplib/_impl/fake/client.py | 16 +++- pubtools/pulplib/_impl/fake/controller.py | 15 ++++ pubtools/pulplib/_impl/fake/match.py | 2 +- pubtools/pulplib/_impl/model/__init__.py | 4 + .../_impl/model/repository/__init__.py | 8 +- .../pulplib/_impl/model/repository/base.py | 85 +++++++++++++++++-- .../_impl/model/repository/container.py | 29 ++++++- .../pulplib/_impl/model/repository/file.py | 17 +++- .../pulplib/_impl/model/repository/yum.py | 79 ++++++++++++++++- setup.py | 2 +- tests/fake/test_fake_sync.py | 49 +++++++++++ tests/repository/test_sync.py | 75 ++++++++++++++++ 14 files changed, 381 insertions(+), 18 deletions(-) create mode 100644 tests/fake/test_fake_sync.py create mode 100644 tests/repository/test_sync.py diff --git a/pubtools/pulplib/__init__.py b/pubtools/pulplib/__init__.py index 2a95db6c..459824f1 100644 --- a/pubtools/pulplib/__init__.py +++ b/pubtools/pulplib/__init__.py @@ -16,6 +16,7 @@ ModulemdDefaultsUnit, Distributor, PublishOptions, + SyncOptions, Task, MaintenanceReport, MaintenanceEntry, diff --git a/pubtools/pulplib/_impl/client/client.py b/pubtools/pulplib/_impl/client/client.py index 65c52558..0caef8e2 100644 --- a/pubtools/pulplib/_impl/client/client.py +++ b/pubtools/pulplib/_impl/client/client.py @@ -229,7 +229,7 @@ def _search( search_type="search", search_options=None, criteria=None, - ): + ): # pylint:disable = too-many-arguments url = os.path.join( self._url, "pulp/api/v2/%s/%s/" % (resource_type, search_type) ) @@ -565,3 +565,18 @@ def map_404_to_none(exception): ) return f_map(response, error_fn=map_404_to_none) + + def _do_sync(self, repo_id, sync_options): + if not sync_options["feed"]: + raise ValueError("Cannot sync with empty feed: '%s'" % sync_options["feed"]) + + url = os.path.join( + self._url, "pulp/api/v2/repositories/%s/actions/sync/" % repo_id + ) + + body = {"override_config": sync_options} + + LOG.debug("Syncing repository %s with feed %s", repo_id, sync_options["feed"]) + return self._task_executor.submit( + self._do_request, method="POST", url=url, json=body + ) diff --git a/pubtools/pulplib/_impl/fake/client.py b/pubtools/pulplib/_impl/fake/client.py index 0960be32..3687979b 100644 --- a/pubtools/pulplib/_impl/fake/client.py +++ b/pubtools/pulplib/_impl/fake/client.py @@ -29,9 +29,10 @@ Publish = namedtuple("Publish", ["repository", "tasks"]) Upload = namedtuple("Upload", ["repository", "tasks", "name", "sha256"]) +Sync = namedtuple("Sync", ["repository", "tasks"]) -class FakeClient(object): +class FakeClient(object): # pylint:disable = too-many-instance-attributes # Client implementation holding data in memory rather than # using a remote Pulp server. # @@ -62,6 +63,7 @@ def __init__(self): self._repo_units = {} self._publish_history = [] self._upload_history = [] + self._sync_history = [] self._maintenance_report = None self._type_ids = self._DEFAULT_TYPE_IDS[:] self._lock = threading.RLock() @@ -356,6 +358,18 @@ def _attach_repo(self, repo): repo._set_client(self) return repo + def _do_sync(self, repo_id, sync_config): # pylint:disable = unused-argument + repo_f = self.get_repository(repo_id) + if repo_f.exception(): + # Repo can't be found, let that exception propagate + return repo_f + + task = Task(id=self._next_task_id(), completed=True, succeeded=True) + + self._sync_history.append(Sync(repo_f.result(), [task])) + + return f_return([task]) + def _next_task_id(self): with self._lock: next_raw_id = self._uuidgen.randint(0, 2 ** 128) diff --git a/pubtools/pulplib/_impl/fake/controller.py b/pubtools/pulplib/_impl/fake/controller.py index ec8921ed..0a9d4194 100644 --- a/pubtools/pulplib/_impl/fake/controller.py +++ b/pubtools/pulplib/_impl/fake/controller.py @@ -112,6 +112,21 @@ def publish_history(self): """ return self.client._publish_history[:] + @property + def sync_history(self): + """A list of repository syncs triggered via this client. + + Each element of this list is a named tuple with the following attributes, + in order: + + ``repository``: + :class:`~pubtools.pulplib.Repository` for which publish was triggered + ``tasks``: + list of :class:`~pubtools.pulplib.Task` generated as a result + of this publish + """ + return self.client._sync_history[:] + @property def upload_history(self): """A list of upload tasks triggered via this client. diff --git a/pubtools/pulplib/_impl/fake/match.py b/pubtools/pulplib/_impl/fake/match.py index 5b4a305f..09515698 100644 --- a/pubtools/pulplib/_impl/fake/match.py +++ b/pubtools/pulplib/_impl/fake/match.py @@ -29,7 +29,7 @@ def wrap(func): return wrap -def match_object(*args, **kwargs): +def match_object(*args, **kwargs): # pylint:disable=inconsistent-return-statements dispatch = args[0] for (klass, func) in CLASS_MATCHERS: if isinstance(dispatch, klass): diff --git a/pubtools/pulplib/_impl/model/__init__.py b/pubtools/pulplib/_impl/model/__init__.py index 8eeb7dc5..9a35e46b 100644 --- a/pubtools/pulplib/_impl/model/__init__.py +++ b/pubtools/pulplib/_impl/model/__init__.py @@ -9,6 +9,10 @@ FileRepository, ContainerImageRepository, PublishOptions, + SyncOptions, + FileSyncOptions, + ContainerSyncOptions, + YumSyncOptions, ) from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit, ModulemdDefaultsUnit from .task import Task diff --git a/pubtools/pulplib/_impl/model/repository/__init__.py b/pubtools/pulplib/_impl/model/repository/__init__.py index 1bdeaf17..7fa39a87 100644 --- a/pubtools/pulplib/_impl/model/repository/__init__.py +++ b/pubtools/pulplib/_impl/model/repository/__init__.py @@ -1,4 +1,4 @@ -from .base import Repository, PublishOptions -from .container import ContainerImageRepository -from .yum import YumRepository -from .file import FileRepository +from .base import Repository, PublishOptions, SyncOptions +from .container import ContainerImageRepository, ContainerSyncOptions +from .yum import YumRepository, YumSyncOptions +from .file import FileRepository, FileSyncOptions diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 27b35a64..3e193aa0 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -1,14 +1,10 @@ import datetime import logging -from attr import validators +from attr import validators, asdict from more_executors.futures import f_proxy -from ..common import ( - PulpObject, - Deletable, - DetachedException, -) +from ..common import PulpObject, Deletable, DetachedException from ..attr import pulp_attrib from ..distributor import Distributor from ..frozenlist import FrozenList @@ -62,6 +58,53 @@ class PublishOptions(object): """ +@attr.s(kw_only=True, frozen=True) +class SyncOptions(object): + """Options controlling a repository + :meth:`~pubtools.pulplib.Repository.sync`. + """ + + feed = pulp_attrib(type=str) + """URL where the repository's content will be synchronized from. + """ + + ssl_validation = pulp_attrib(default=None, type=bool) + """Indicates if the server's SSL certificate is verified against the CA certificate uploaded. + """ + + ssl_ca_cert = pulp_attrib(default=None, type=str) + """CA certificate string used to validate the feed source's SSL certificate + """ + + ssl_client_cert = pulp_attrib(default=None, type=str) + """Certificate used as the client certificate when synchronizing the repository + """ + + ssl_client_key = pulp_attrib(default=None, type=str) + """Private key to the certificate specified in ssl_client_cert + """ + + max_speed = pulp_attrib(default=None, type=int) + """Representing the maximum speed that the importer should be allowed to transfer + """ + + proxy_host = pulp_attrib(default=None, type=str) + """A string representing the URL of the proxy server that should be used when synchronizing + """ + + proxy_port = pulp_attrib(default=None, type=int) + """An integer representing the port that should be used when connecting to proxy_host. + """ + + proxy_username = pulp_attrib(default=None, type=str) + """A string representing the username that should be used to authenticate with the proxy server + """ + + proxy_password = pulp_attrib(default=None, type=str) + """A string representing the password that should be used to authenticate with the proxy server + """ + + @attr.s(kw_only=True, frozen=True) class Repository(PulpObject, Deletable): """Represents a Pulp repository.""" @@ -168,7 +211,7 @@ def _check_repo_id(self, _, value): for distributor in value: if not distributor.repo_id: return - elif distributor.repo_id == self.id: + if distributor.repo_id == self.id: return raise ValueError( "repo_id doesn't match for %s. repo_id: %s, distributor.repo_id: %s" @@ -343,6 +386,34 @@ def publish(self, options=PublishOptions()): return f_proxy(self._client._publish_repository(self, to_publish)) + def sync(self, options=SyncOptions(feed="")): + """Sync repository with feed + + Args: + options (SyncOptions) + Options used to customize the behavior of sync process. + If omitted, the Pulp server's defaults apply. + + Returns: + Future[list[:class:`~pubtools.pulplib.Task`]] + A future which is resolved when publish succeeds. + + The future contains a list of zero or more tasks triggered and awaited + during the publish operation. + + Raises: + DetachedException + If this instance is not attached to a Pulp client. + """ + if not self._client: + raise DetachedException() + + return f_proxy( + self._client._do_sync( + self.id, asdict(options, filter=lambda name, val: val is not None) + ) + ) + def remove_content(self, **kwargs): """Remove all content of requested types from this repository. diff --git a/pubtools/pulplib/_impl/model/repository/container.py b/pubtools/pulplib/_impl/model/repository/container.py index 5005155f..db183717 100644 --- a/pubtools/pulplib/_impl/model/repository/container.py +++ b/pubtools/pulplib/_impl/model/repository/container.py @@ -1,8 +1,35 @@ -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..attr import pulp_attrib from ... import compat_attr as attr +@attr.s(kw_only=True, frozen=True) +class ContainerSyncOptions(SyncOptions): + """Options controlling a container repository + :meth:`~pubtools.pulplib.ContainerImageRepository.sync`. + """ + + upstream_name = pulp_attrib(default=None, type=str) + """The name of the repository to import from the upstream repository. + """ + + tags = pulp_attrib(default=None, type=str) + """List of tags to include on sync. + """ + + enable_v1 = pulp_attrib(default=False, type=bool) + """Boolean to control whether to attempt using registry API v1 during synchronization. + + Default is False + """ + + enable_v2 = pulp_attrib(default=False, type=bool) + """Boolean to control whether to attempt using registry API v2 during synchronization. + + Default is True + """ + + @repo_type("docker-repo") @attr.s(kw_only=True, frozen=True) class ContainerImageRepository(Repository): diff --git a/pubtools/pulplib/_impl/model/repository/file.py b/pubtools/pulplib/_impl/model/repository/file.py index 0606efba..d732622b 100644 --- a/pubtools/pulplib/_impl/model/repository/file.py +++ b/pubtools/pulplib/_impl/model/repository/file.py @@ -4,7 +4,7 @@ from attr import validators from more_executors.futures import f_flat_map, f_map, f_proxy -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..frozenlist import FrozenList from ..attr import pulp_attrib from ..common import DetachedException @@ -14,6 +14,21 @@ LOG = logging.getLogger("pubtools.pulplib") +@attr.s(kw_only=True, frozen=True) +class FileSyncOptions(SyncOptions): + """Options controlling a file repository + :meth:`~pubtools.pulplib.FileRepository.sync`. + """ + + remove_missing = pulp_attrib(default=False, type=bool) + """If true, as the repository is synchronized, old rpms will be removed. + """ + + validate = pulp_attrib(default=None, type=bool) + """If True, checksum of each file will be verified against the metadata's expectation + """ + + @repo_type("iso-repo") @attr.s(kw_only=True, frozen=True) class FileRepository(Repository): diff --git a/pubtools/pulplib/_impl/model/repository/yum.py b/pubtools/pulplib/_impl/model/repository/yum.py index 6057ea65..bb63cf7c 100644 --- a/pubtools/pulplib/_impl/model/repository/yum.py +++ b/pubtools/pulplib/_impl/model/repository/yum.py @@ -1,9 +1,86 @@ -from .base import Repository, repo_type +from .base import Repository, SyncOptions, repo_type from ..frozenlist import FrozenList from ..attr import pulp_attrib from ... import compat_attr as attr +@attr.s(kw_only=True, frozen=True) +class YumSyncOptions(SyncOptions): + """Options controlling a container repository + :meth:`~pubtools.pulplib.YumRepository.sync`. + """ + + query_auth_token = pulp_attrib(default=None, type=str) + """An authorization token that will be added to every request made to the feed URL's server + """ + + validate = pulp_attrib(default=None, type=bool) + """If True, checksum of each file will be verified against the metadata's expectation + """ + + max_downloads = pulp_attrib(default=None, type=int) + """Number of threads used when synchronizing the repository. + """ + + remove_missing = pulp_attrib(default=None, type=bool) + """If true, as the repository is synchronized, old rpms will be removed. + """ + + retain_old_count = pulp_attrib(default=None, type=int) + """Count indicating how many old rpm versions to retain. + """ + + skip = pulp_attrib(default=None, type=list) + """List of content types to be skipped during the repository synchronization + """ + + checksum_type = pulp_attrib(default=None, type=str) + """checksum type to use for metadata generation. + + Defaults to source checksum type of sha256 + """ + + num_retries = pulp_attrib(default=None, type=int) + """Number of times to retry before declaring an error during repository synchronization + + Default is to 2. + """ + + copy_children = pulp_attrib(default=None, type=bool) + """when False, will not attempt to locate and copy child packages of errata, groups, or categories + """ + + download_policy = pulp_attrib(default=None, type=str) + """Set the download policy for a repository. + + Supported options are immediate,on_demand,background + """ + + force_full = pulp_attrib(default=None, type=bool) + """Boolean flag. If true, full re-sync is triggered. + """ + + require_signature = pulp_attrib(default=None, type=bool) + """Requires that imported packages like RPM/DRPM/SRPM should be signed + """ + + allowed_keys = pulp_attrib(default=None, type=str) + """Comma-separated list of allowed signature key IDs that imported packages can be signed with + """ + + recursive = pulp_attrib(default=None, type=bool) + """If true, units are copied together with the latest versions of their dependencies + """ + + recursive_conservative = pulp_attrib(default=None, type=bool) + """If true, units are copied together with their dependencies, unless those are already satisfied by the content in the target repository. + """ + + additional_repos = pulp_attrib(default=None, type=bool) + """This option allows for dependencies to be found in repositories ouside of the one the specified in the copy command + """ + + @repo_type("rpm-repo") @attr.s(kw_only=True, frozen=True) class YumRepository(Repository): diff --git a/setup.py b/setup.py index 5feb4896..b2fde7fd 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def get_requirements(): setup( name="pubtools-pulplib", - version="2.4.0", + version="2.5.0", packages=find_packages(exclude=["tests"]), package_data={"pubtools.pulplib._impl.schema": ["*.yaml"]}, url="https://github.com/release-engineering/pubtools-pulplib", diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py new file mode 100644 index 00000000..a6664729 --- /dev/null +++ b/tests/fake/test_fake_sync.py @@ -0,0 +1,49 @@ +from pubtools.pulplib import FakeController, YumRepository, SyncOptions, PulpException + + +def test_can_sync(): + """repo.sync() succeeds with fake client and populates sync_history.""" + controller = FakeController() + + controller.insert_repository(YumRepository(id="repo1")) + controller.insert_repository(YumRepository(id="repo2")) + + client = controller.client + repo1 = client.get_repository("repo1") + + # Call to sync should succeed + sync_f = repo1.sync(SyncOptions(feed="mock://feed/")) + + # The future should resolve successfully + tasks = sync_f.result() + + # It should have returned at least one successful task. + assert tasks + for task in tasks: + assert task.succeeded + + # The change should be reflected in the controller's publish history + history = controller.sync_history + + assert len(history) == 1 + assert history[0].repository.id == "repo1" + assert history[0].tasks == tasks + + +def test_publish_absent_raises(): + """repo.publish() of a nonexistent repo raises.""" + controller = FakeController() + + controller.insert_repository(YumRepository(id="repo1")) + + client = controller.client + repo_copy1 = client.get_repository("repo1") + repo_copy2 = client.get_repository("repo1") + + # If I delete the repo through one handle... + assert repo_copy1.delete().result() + + # ...then publish through the other handle becomes impossible + exception = repo_copy2.sync(SyncOptions(feed="mock://feed/")).exception() + assert isinstance(exception, PulpException) + assert "repo1 not found" in str(exception) diff --git a/tests/repository/test_sync.py b/tests/repository/test_sync.py new file mode 100644 index 00000000..2c805f65 --- /dev/null +++ b/tests/repository/test_sync.py @@ -0,0 +1,75 @@ +import logging +import pytest + +from pubtools.pulplib import ( + Repository, + YumRepository, + Task, + Distributor, + DetachedException, + SyncOptions, + TaskFailedException, +) + + +@pytest.fixture +def fixture_sync_async_response(requests_mocker): + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/actions/sync/", + [{"json": {"spawned_tasks": [{"task_id": "task1"}]}}], + ) + + +@pytest.fixture +def fixture_search_task_response(requests_mocker): + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/tasks/search/", + [{"json": [{"task_id": "task1", "state": "finished"}]}], + ) + + +def test_detached(): + """publish raises if called on a detached repo""" + with pytest.raises(DetachedException): + Repository(id="some-repo").sync() + + +def test_sync( + fast_poller, + client, + requests_mocker, + fixture_sync_async_response, + fixture_search_task_response, +): + """publish succeeds and returns tasks from each applicable distributor""" + repo = YumRepository(id="some-repo") + repo.__dict__["_client"] = client + + # empty options should fail as feed is required to be non-empty + try: + repo.sync().result() + assert "Exception should have been raised" + except ValueError: + pass + + +def test_sync_with_options( + requests_mocker, client, fixture_sync_async_response, fixture_search_task_response +): + """publish passes expected config into distributors based on publish options""" + repo = YumRepository(id="some-repo") + repo.__dict__["_client"] = client + + options = SyncOptions(ssl_validation=False, feed="mock://example.com/") + + # It should have succeeded, with the tasks as retrieved from Pulp + assert repo.sync(options).result() == [ + Task(id="task1", succeeded=True, completed=True) + ] + + req = requests_mocker.request_history + + assert req[0].json()["override_config"] == { + "ssl_validation": False, + "feed": "mock://example.com/", + } From 46a8bcb862f26829b392247a012c97934570830f Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:23:12 +0100 Subject: [PATCH 02/19] Update pubtools/pulplib/_impl/model/repository/base.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/model/repository/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 3e193aa0..3241ccd0 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -396,7 +396,7 @@ def sync(self, options=SyncOptions(feed="")): Returns: Future[list[:class:`~pubtools.pulplib.Task`]] - A future which is resolved when publish succeeds. + A future which is resolved when sync succeeds. The future contains a list of zero or more tasks triggered and awaited during the publish operation. From 90c1034affdad19e7bcceaa436ba900b607a7b58 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:23:37 +0100 Subject: [PATCH 03/19] Update pubtools/pulplib/_impl/model/repository/base.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/model/repository/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 3241ccd0..5af61e49 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -399,7 +399,7 @@ def sync(self, options=SyncOptions(feed="")): A future which is resolved when sync succeeds. The future contains a list of zero or more tasks triggered and awaited - during the publish operation. + during the sync operation. Raises: DetachedException From 117aa496ae930493f86d2bdee71d1ab56af03a73 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:24:00 +0100 Subject: [PATCH 04/19] Update tests/fake/test_fake_sync.py Co-Authored-By: Rohan McGovern --- tests/fake/test_fake_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py index a6664729..4897b4ff 100644 --- a/tests/fake/test_fake_sync.py +++ b/tests/fake/test_fake_sync.py @@ -43,7 +43,7 @@ def test_publish_absent_raises(): # If I delete the repo through one handle... assert repo_copy1.delete().result() - # ...then publish through the other handle becomes impossible + # ...then sync through the other handle becomes impossible exception = repo_copy2.sync(SyncOptions(feed="mock://feed/")).exception() assert isinstance(exception, PulpException) assert "repo1 not found" in str(exception) From 9493d862a961aea09b5c81f8690d888fcd3f5413 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:24:18 +0100 Subject: [PATCH 05/19] Update tests/repository/test_sync.py Co-Authored-By: Rohan McGovern --- tests/repository/test_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/repository/test_sync.py b/tests/repository/test_sync.py index 2c805f65..bf67d50d 100644 --- a/tests/repository/test_sync.py +++ b/tests/repository/test_sync.py @@ -29,7 +29,7 @@ def fixture_search_task_response(requests_mocker): def test_detached(): - """publish raises if called on a detached repo""" + """sync raises if called on a detached repo""" with pytest.raises(DetachedException): Repository(id="some-repo").sync() From b19e954a347a8059be06197ac583e4a6a803df26 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:49:45 +0100 Subject: [PATCH 06/19] Update pubtools/pulplib/_impl/model/repository/container.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/model/repository/container.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pubtools/pulplib/_impl/model/repository/container.py b/pubtools/pulplib/_impl/model/repository/container.py index db183717..eaffb860 100644 --- a/pubtools/pulplib/_impl/model/repository/container.py +++ b/pubtools/pulplib/_impl/model/repository/container.py @@ -11,6 +11,7 @@ class ContainerSyncOptions(SyncOptions): upstream_name = pulp_attrib(default=None, type=str) """The name of the repository to import from the upstream repository. + For example, if syncing from repository `quay.io/fedora/fedora`, upstream_name should be set to `fedora/fedora`. """ tags = pulp_attrib(default=None, type=str) From 665d3ddedd1c76dcaead65180f1563bf3646c5cd Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:52:00 +0100 Subject: [PATCH 07/19] Update pubtools/pulplib/_impl/model/repository/file.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/model/repository/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubtools/pulplib/_impl/model/repository/file.py b/pubtools/pulplib/_impl/model/repository/file.py index d732622b..4b9a5439 100644 --- a/pubtools/pulplib/_impl/model/repository/file.py +++ b/pubtools/pulplib/_impl/model/repository/file.py @@ -21,7 +21,7 @@ class FileSyncOptions(SyncOptions): """ remove_missing = pulp_attrib(default=False, type=bool) - """If true, as the repository is synchronized, old rpms will be removed. + """If true, as the repository is synchronized, old files will be removed. """ validate = pulp_attrib(default=None, type=bool) From 3f174eee6e19ae6bbd4eecb2356a57347ecc64c7 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:54:37 +0100 Subject: [PATCH 08/19] Update tests/fake/test_fake_sync.py Co-Authored-By: Rohan McGovern --- tests/fake/test_fake_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py index 4897b4ff..497cd90f 100644 --- a/tests/fake/test_fake_sync.py +++ b/tests/fake/test_fake_sync.py @@ -22,7 +22,7 @@ def test_can_sync(): for task in tasks: assert task.succeeded - # The change should be reflected in the controller's publish history + # The change should be reflected in the controller's sync history history = controller.sync_history assert len(history) == 1 From ed53060bb022920a3be602ad89cfb3a5224b37fc Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:55:46 +0100 Subject: [PATCH 09/19] Update tests/fake/test_fake_sync.py Co-Authored-By: Rohan McGovern --- tests/fake/test_fake_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py index 497cd90f..67fdf348 100644 --- a/tests/fake/test_fake_sync.py +++ b/tests/fake/test_fake_sync.py @@ -30,7 +30,7 @@ def test_can_sync(): assert history[0].tasks == tasks -def test_publish_absent_raises(): +def test_sync_absent_raises(): """repo.publish() of a nonexistent repo raises.""" controller = FakeController() From 80a38878076d4261f07ca1ff0bb9112f3399d5b0 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 08:56:27 +0100 Subject: [PATCH 10/19] Update tests/fake/test_fake_sync.py Co-Authored-By: Rohan McGovern --- tests/fake/test_fake_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py index 67fdf348..6e9c3463 100644 --- a/tests/fake/test_fake_sync.py +++ b/tests/fake/test_fake_sync.py @@ -31,7 +31,7 @@ def test_can_sync(): def test_sync_absent_raises(): - """repo.publish() of a nonexistent repo raises.""" + """repo.sync() of a nonexistent repo raises.""" controller = FakeController() controller.insert_repository(YumRepository(id="repo1")) From 54c2cadf2adeb8f98f902e7b5a61eeb8f27f6d5f Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 13:43:10 +0100 Subject: [PATCH 11/19] - Added sync config to fake sync history - Fixed documentation for SyncOptions attributes - Fixed wrong attribute types - Renamed test_sync to test_sync_no_feed - Fixed doc strings for tests --- pubtools/pulplib/_impl/fake/client.py | 4 ++-- pubtools/pulplib/_impl/model/repository/base.py | 4 +++- pubtools/pulplib/_impl/model/repository/container.py | 2 +- pubtools/pulplib/_impl/model/repository/yum.py | 4 ++-- tests/repository/test_sync.py | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pubtools/pulplib/_impl/fake/client.py b/pubtools/pulplib/_impl/fake/client.py index 3687979b..21865b27 100644 --- a/pubtools/pulplib/_impl/fake/client.py +++ b/pubtools/pulplib/_impl/fake/client.py @@ -29,7 +29,7 @@ Publish = namedtuple("Publish", ["repository", "tasks"]) Upload = namedtuple("Upload", ["repository", "tasks", "name", "sha256"]) -Sync = namedtuple("Sync", ["repository", "tasks"]) +Sync = namedtuple("Sync", ["repository", "tasks", "sync_config"]) class FakeClient(object): # pylint:disable = too-many-instance-attributes @@ -366,7 +366,7 @@ def _do_sync(self, repo_id, sync_config): # pylint:disable = unused-argument task = Task(id=self._next_task_id(), completed=True, succeeded=True) - self._sync_history.append(Sync(repo_f.result(), [task])) + self._sync_history.append(Sync(repo_f.result(), [task], sync_config)) return f_return([task]) diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 5af61e49..054ce6e8 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -85,7 +85,9 @@ class SyncOptions(object): """ max_speed = pulp_attrib(default=None, type=int) - """Representing the maximum speed that the importer should be allowed to transfer + """The maximum download speed in bytes/sec for a task (such as a sync). + + Default is None """ proxy_host = pulp_attrib(default=None, type=str) diff --git a/pubtools/pulplib/_impl/model/repository/container.py b/pubtools/pulplib/_impl/model/repository/container.py index eaffb860..4a04c692 100644 --- a/pubtools/pulplib/_impl/model/repository/container.py +++ b/pubtools/pulplib/_impl/model/repository/container.py @@ -14,7 +14,7 @@ class ContainerSyncOptions(SyncOptions): For example, if syncing from repository `quay.io/fedora/fedora`, upstream_name should be set to `fedora/fedora`. """ - tags = pulp_attrib(default=None, type=str) + tags = pulp_attrib(default=None, type=list) """List of tags to include on sync. """ diff --git a/pubtools/pulplib/_impl/model/repository/yum.py b/pubtools/pulplib/_impl/model/repository/yum.py index bb63cf7c..85aa7aee 100644 --- a/pubtools/pulplib/_impl/model/repository/yum.py +++ b/pubtools/pulplib/_impl/model/repository/yum.py @@ -64,8 +64,8 @@ class YumSyncOptions(SyncOptions): """Requires that imported packages like RPM/DRPM/SRPM should be signed """ - allowed_keys = pulp_attrib(default=None, type=str) - """Comma-separated list of allowed signature key IDs that imported packages can be signed with + allowed_keys = pulp_attrib(default=None, type=list) + """List of allowed signature key IDs that imported packages can be signed with """ recursive = pulp_attrib(default=None, type=bool) diff --git a/tests/repository/test_sync.py b/tests/repository/test_sync.py index bf67d50d..df65a81c 100644 --- a/tests/repository/test_sync.py +++ b/tests/repository/test_sync.py @@ -34,14 +34,14 @@ def test_detached(): Repository(id="some-repo").sync() -def test_sync( +def test_sync_no_feed( fast_poller, client, requests_mocker, fixture_sync_async_response, fixture_search_task_response, ): - """publish succeeds and returns tasks from each applicable distributor""" + """Test sync fail as no feed is provided.""" repo = YumRepository(id="some-repo") repo.__dict__["_client"] = client @@ -56,7 +56,7 @@ def test_sync( def test_sync_with_options( requests_mocker, client, fixture_sync_async_response, fixture_search_task_response ): - """publish passes expected config into distributors based on publish options""" + """Test sync passes, test whether sync options are passed to override config.""" repo = YumRepository(id="some-repo") repo.__dict__["_client"] = client From a385ae1ce3d9aaefed526d12bd351e05a6db4ec9 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 14:06:28 +0100 Subject: [PATCH 12/19] Removed unneeded SyncOptions attributes --- .../pulplib/_impl/model/repository/file.py | 4 ---- .../pulplib/_impl/model/repository/yum.py | 20 ------------------- 2 files changed, 24 deletions(-) diff --git a/pubtools/pulplib/_impl/model/repository/file.py b/pubtools/pulplib/_impl/model/repository/file.py index 4b9a5439..5e5090f6 100644 --- a/pubtools/pulplib/_impl/model/repository/file.py +++ b/pubtools/pulplib/_impl/model/repository/file.py @@ -24,10 +24,6 @@ class FileSyncOptions(SyncOptions): """If true, as the repository is synchronized, old files will be removed. """ - validate = pulp_attrib(default=None, type=bool) - """If True, checksum of each file will be verified against the metadata's expectation - """ - @repo_type("iso-repo") @attr.s(kw_only=True, frozen=True) diff --git a/pubtools/pulplib/_impl/model/repository/yum.py b/pubtools/pulplib/_impl/model/repository/yum.py index 85aa7aee..450aa558 100644 --- a/pubtools/pulplib/_impl/model/repository/yum.py +++ b/pubtools/pulplib/_impl/model/repository/yum.py @@ -14,10 +14,6 @@ class YumSyncOptions(SyncOptions): """An authorization token that will be added to every request made to the feed URL's server """ - validate = pulp_attrib(default=None, type=bool) - """If True, checksum of each file will be verified against the metadata's expectation - """ - max_downloads = pulp_attrib(default=None, type=int) """Number of threads used when synchronizing the repository. """ @@ -46,10 +42,6 @@ class YumSyncOptions(SyncOptions): Default is to 2. """ - copy_children = pulp_attrib(default=None, type=bool) - """when False, will not attempt to locate and copy child packages of errata, groups, or categories - """ - download_policy = pulp_attrib(default=None, type=str) """Set the download policy for a repository. @@ -68,18 +60,6 @@ class YumSyncOptions(SyncOptions): """List of allowed signature key IDs that imported packages can be signed with """ - recursive = pulp_attrib(default=None, type=bool) - """If true, units are copied together with the latest versions of their dependencies - """ - - recursive_conservative = pulp_attrib(default=None, type=bool) - """If true, units are copied together with their dependencies, unless those are already satisfied by the content in the target repository. - """ - - additional_repos = pulp_attrib(default=None, type=bool) - """This option allows for dependencies to be found in repositories ouside of the one the specified in the copy command - """ - @repo_type("rpm-repo") @attr.s(kw_only=True, frozen=True) From e578582776a31ed362f4a938eb8c8149061ec307 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 14:10:46 +0100 Subject: [PATCH 13/19] Removed enable_v1, enable_v2 options --- pubtools/pulplib/_impl/model/repository/container.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pubtools/pulplib/_impl/model/repository/container.py b/pubtools/pulplib/_impl/model/repository/container.py index 4a04c692..6b1c71b8 100644 --- a/pubtools/pulplib/_impl/model/repository/container.py +++ b/pubtools/pulplib/_impl/model/repository/container.py @@ -18,18 +18,6 @@ class ContainerSyncOptions(SyncOptions): """List of tags to include on sync. """ - enable_v1 = pulp_attrib(default=False, type=bool) - """Boolean to control whether to attempt using registry API v1 during synchronization. - - Default is False - """ - - enable_v2 = pulp_attrib(default=False, type=bool) - """Boolean to control whether to attempt using registry API v2 during synchronization. - - Default is True - """ - @repo_type("docker-repo") @attr.s(kw_only=True, frozen=True) From aa040a86a068ece4526ca2287b55fb6802669e86 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Thu, 13 Feb 2020 15:32:56 +0100 Subject: [PATCH 14/19] - Remove SyncOptions from publically visible classes - Make ContainerSyncOptions, FileSyncOptions, YumSyncOptions public --- pubtools/pulplib/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pubtools/pulplib/__init__.py b/pubtools/pulplib/__init__.py index 459824f1..bfa2defa 100644 --- a/pubtools/pulplib/__init__.py +++ b/pubtools/pulplib/__init__.py @@ -16,7 +16,9 @@ ModulemdDefaultsUnit, Distributor, PublishOptions, - SyncOptions, + FileSyncOptions, + ContainerSyncOptions, + YumSyncOptions, Task, MaintenanceReport, MaintenanceEntry, From 13bd18117ef84a01305347b8d09bd7e99762d698 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Fri, 14 Feb 2020 15:36:06 +0100 Subject: [PATCH 15/19] Fixed wrong imports --- tests/fake/test_fake_sync.py | 11 ++++++++--- tests/repository/test_sync.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/fake/test_fake_sync.py b/tests/fake/test_fake_sync.py index 6e9c3463..c42f6a0f 100644 --- a/tests/fake/test_fake_sync.py +++ b/tests/fake/test_fake_sync.py @@ -1,4 +1,9 @@ -from pubtools.pulplib import FakeController, YumRepository, SyncOptions, PulpException +from pubtools.pulplib import ( + FakeController, + YumRepository, + YumSyncOptions, + PulpException, +) def test_can_sync(): @@ -12,7 +17,7 @@ def test_can_sync(): repo1 = client.get_repository("repo1") # Call to sync should succeed - sync_f = repo1.sync(SyncOptions(feed="mock://feed/")) + sync_f = repo1.sync(YumSyncOptions(feed="mock://feed/")) # The future should resolve successfully tasks = sync_f.result() @@ -44,6 +49,6 @@ def test_sync_absent_raises(): assert repo_copy1.delete().result() # ...then sync through the other handle becomes impossible - exception = repo_copy2.sync(SyncOptions(feed="mock://feed/")).exception() + exception = repo_copy2.sync(YumSyncOptions(feed="mock://feed/")).exception() assert isinstance(exception, PulpException) assert "repo1 not found" in str(exception) diff --git a/tests/repository/test_sync.py b/tests/repository/test_sync.py index df65a81c..691a82e7 100644 --- a/tests/repository/test_sync.py +++ b/tests/repository/test_sync.py @@ -7,7 +7,7 @@ Task, Distributor, DetachedException, - SyncOptions, + YumSyncOptions, TaskFailedException, ) @@ -60,7 +60,7 @@ def test_sync_with_options( repo = YumRepository(id="some-repo") repo.__dict__["_client"] = client - options = SyncOptions(ssl_validation=False, feed="mock://example.com/") + options = YumSyncOptions(ssl_validation=False, feed="mock://example.com/") # It should have succeeded, with the tasks as retrieved from Pulp assert repo.sync(options).result() == [ From 55d50ea7d4ac232076032eb87b9eabccc74a6da6 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Tue, 18 Feb 2020 13:24:47 +0100 Subject: [PATCH 16/19] Update pubtools/pulplib/_impl/fake/controller.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/fake/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubtools/pulplib/_impl/fake/controller.py b/pubtools/pulplib/_impl/fake/controller.py index 0a9d4194..aecf2a45 100644 --- a/pubtools/pulplib/_impl/fake/controller.py +++ b/pubtools/pulplib/_impl/fake/controller.py @@ -120,7 +120,7 @@ def sync_history(self): in order: ``repository``: - :class:`~pubtools.pulplib.Repository` for which publish was triggered + :class:`~pubtools.pulplib.Repository` for which sync was triggered ``tasks``: list of :class:`~pubtools.pulplib.Task` generated as a result of this publish From d207a3e03be4e8e20b560a84d3052fafcece5e2e Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Tue, 18 Feb 2020 13:24:58 +0100 Subject: [PATCH 17/19] Update pubtools/pulplib/_impl/fake/controller.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/fake/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubtools/pulplib/_impl/fake/controller.py b/pubtools/pulplib/_impl/fake/controller.py index aecf2a45..fddf834e 100644 --- a/pubtools/pulplib/_impl/fake/controller.py +++ b/pubtools/pulplib/_impl/fake/controller.py @@ -123,7 +123,7 @@ def sync_history(self): :class:`~pubtools.pulplib.Repository` for which sync was triggered ``tasks``: list of :class:`~pubtools.pulplib.Task` generated as a result - of this publish + of this sync """ return self.client._sync_history[:] From 40fbdea5be7f2bceb23cbe35bffbc97e6fe31024 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Tue, 18 Feb 2020 13:25:21 +0100 Subject: [PATCH 18/19] Update pubtools/pulplib/_impl/fake/controller.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/_impl/fake/controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pubtools/pulplib/_impl/fake/controller.py b/pubtools/pulplib/_impl/fake/controller.py index fddf834e..3b85b2d2 100644 --- a/pubtools/pulplib/_impl/fake/controller.py +++ b/pubtools/pulplib/_impl/fake/controller.py @@ -124,6 +124,8 @@ def sync_history(self): ``tasks``: list of :class:`~pubtools.pulplib.Task` generated as a result of this sync + ``sync_config``: + :class:`~pubtools.pulplib.SyncConfig` (of the appropriate subclass) used for this sync """ return self.client._sync_history[:] From cd2bda0c0bbfc5c83dbab79f355eed7c4f0b03a3 Mon Sep 17 00:00:00 2001 From: Jindrich Luza Date: Tue, 18 Feb 2020 13:25:57 +0100 Subject: [PATCH 19/19] Update pubtools/pulplib/__init__.py Co-Authored-By: Rohan McGovern --- pubtools/pulplib/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pubtools/pulplib/__init__.py b/pubtools/pulplib/__init__.py index bfa2defa..3399acb2 100644 --- a/pubtools/pulplib/__init__.py +++ b/pubtools/pulplib/__init__.py @@ -19,6 +19,7 @@ FileSyncOptions, ContainerSyncOptions, YumSyncOptions, + SyncOptions, Task, MaintenanceReport, MaintenanceEntry,