diff --git a/docs/api-reference/stages.rst b/docs/api-reference/stages.rst index 60f3e711cf..910bfa0a92 100644 --- a/docs/api-reference/stages.rst +++ b/docs/api-reference/stages.rst @@ -24,7 +24,7 @@ DeclarativeVersion .. autoclass:: pulpcore.plugin.stages.DeclarativeContent :no-members: - :members: pipeline_stages + :members: get_or_create_future .. _stages-api: @@ -47,16 +47,12 @@ Artifact Related Stages ^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: pulpcore.plugin.stages.ArtifactDownloader - :special-members: __call__ .. autoclass:: pulpcore.plugin.stages.ArtifactSaver - :special-members: __call__ .. autoclass:: pulpcore.plugin.stages.RemoteArtifactSaver - :special-members: __call__ .. autoclass:: pulpcore.plugin.stages.QueryExistingArtifacts - :special-members: __call__ .. _content-stages: @@ -65,14 +61,11 @@ Content Related Stages ^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: pulpcore.plugin.stages.ContentSaver - :special-members: __call__ :private-members: _pre_save, _post_save .. autoclass:: pulpcore.plugin.stages.QueryExistingContents - :special-members: __call__ .. autoclass:: pulpcore.plugin.stages.ResolveContentFutures - :special-members: __call__ .. _content-association-stages: @@ -80,9 +73,9 @@ Content Related Stages Content Association and Unassociation Stages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. autoclass:: pulpcore.plugin.stages.RemoveDuplicates + .. autoclass:: pulpcore.plugin.stages.ContentAssociation - :special-members: __call__ .. autoclass:: pulpcore.plugin.stages.ContentUnassociation - :special-members: __call__ diff --git a/docs/plugin-writer/concepts/subclassing/serializers.rst b/docs/plugin-writer/concepts/subclassing/serializers.rst index 31553856fa..a3829c66b9 100644 --- a/docs/plugin-writer/concepts/subclassing/serializers.rst +++ b/docs/plugin-writer/concepts/subclassing/serializers.rst @@ -34,7 +34,7 @@ serializer needs to add that field as well. ) class Meta: - fields = ContentSerializer.Meta.fields + ('relative_path',) + fields = SingleArtifactContentSerializer.Meta.fields + ('relative_path',) model = FileContent Help Text diff --git a/docs/plugin-writer/concepts/tasks/add-remove.rst b/docs/plugin-writer/concepts/tasks/add-remove.rst index c8f2966d5c..dfb6fb82c2 100644 --- a/docs/plugin-writer/concepts/tasks/add-remove.rst +++ b/docs/plugin-writer/concepts/tasks/add-remove.rst @@ -3,7 +3,7 @@ Adding and Removing Content For adding and removing content, Pulp 3 provides a layered plugin API. The docs below explain our lower level API; this information is helpful to understand how a synchronize task works under the -hood. **For implementation, it is highly recommended to use** :ref:`stages-docs` **instead**. +hood. Repository Versions ------------------- @@ -33,6 +33,10 @@ working directory setup, and database cleanup after encountering failures. Synchronizing ------------- +.. tip:: + + Please consider using the high level :ref:`stages-docs` for actual implementations. + Most plugins will define a synchronize task, which fetches content from a remote repository, and adds it to a Pulp repository. diff --git a/pulpcore/plugin/stages/api.py b/pulpcore/plugin/stages/api.py index 60f88cf36f..12e2b88f19 100644 --- a/pulpcore/plugin/stages/api.py +++ b/pulpcore/plugin/stages/api.py @@ -230,6 +230,11 @@ class EndStage(Stage): """ async def __call__(self): + """ + This method drains items from the last queue and drops them. + + Importantly it does not try to put items into the nonexistent next queue. + """ # We overwrite __call__ here to avoid trying to put None in `self._out_q`. async for _ in self.items(): # noqa pass diff --git a/pulpcore/plugin/stages/association_stages.py b/pulpcore/plugin/stages/association_stages.py index f86194306d..b64744cc8b 100644 --- a/pulpcore/plugin/stages/association_stages.py +++ b/pulpcore/plugin/stages/association_stages.py @@ -91,7 +91,8 @@ class RemoveDuplicates(Stage): """ Stage allows plugins to remove content that would break repository uniqueness constraints. - This stage is expected to be added by the DeclarativeVersion. See that class for example usage. + This stage is expected to be added by the + :class:`~pulpcore.plugin.stages.DeclarativeVersion`. See that class for example usage. """ def __init__(self, new_version, model, field_names): diff --git a/pulpcore/plugin/stages/declarative_version.py b/pulpcore/plugin/stages/declarative_version.py index be3de0f0fc..155e17f5d6 100644 --- a/pulpcore/plugin/stages/declarative_version.py +++ b/pulpcore/plugin/stages/declarative_version.py @@ -29,14 +29,31 @@ def __init__(self, first_stage, repository, mirror=True, download_artifacts=True The pipeline stages perform the following steps by default: 1. Create the new :class:`~pulpcore.plugin.models.RepositoryVersion` - 2. Query existing artifacts to determine which are already local to Pulp - 3. Download any undownloaded :class:`~pulpcore.plugin.models.Artifact` objects. - 4. Save the newly downloaded :class:`~pulpcore.plugin.models.Artifact` objects - 5. Query for Content units already present in Pulp - 6. Save new Content units not yet present in Pulp - 7. Associate all content units with the new - :class:`~pulpcore.plugin.models.RepositoryVersion`. - 8. Unassociate any content units not declared in the stream (only when sync_mode='mirror') + 2. Use the provided `first_stage` to construct + :class:`~pulpcore.plugin.stages.DeclarativeContent` + 3. Query existing artifacts to determine which are already local to Pulp with + :class:`~pulpcore.plugin.stages.QueryExistingArtifacts` + 4. Download any undownloaded :class:`~pulpcore.plugin.models.Artifact` objects with + :class:`~pulpcore.plugin.stages.ArtifactDownloader` + 5. Save the newly downloaded :class:`~pulpcore.plugin.models.Artifact` objects with + :class:`~pulpcore.plugin.stages.ArtifactSaver` + 6. Query for Content units already present in Pulp with + :class:`~pulpcore.plugin.stages.QueryExistingContents` + 7. Save new Content units not yet present in Pulp with + :class:`~pulpcore.plugin.stages.ContentSaver` + 8. Attach :class:`~pulpcore.plugin.models.RemoteArtifact` to the + :class:`~pulpcore.plugin.models.Content` via + :class:`~pulpcore.plugin.stages.RemoteArtifactSaver` + 9. Resolve the attached :class:`~asyncio.Future` of + :class:`~pulpcore.plugin.stages.DeclarativeContent` with + :class:`~pulpcore.plugin.stages.ResolveContentFutures` + 10. Remove duplicate content in the repository version if `remove_duplicates` is given by + :class:`~pulpcore.plugin.stages.RemoveDuplicates` + 11. Associate all content units with the new + :class:`~pulpcore.plugin.models.RepositoryVersion` with + :class:`~pulpcore.plugin.stages.ContentAssociation` + 12. Unassociate any content units not declared in the stream (only when sync_mode='mirror') + with :class:`~pulpcore.plugin.stages.ContentUnassociation` To do this, the plugin writer should subclass the :class:`~pulpcore.plugin.stages.Stage` class and define its