Improve, cleanup API-docs

re #4406
https://pulp.plan.io/issues/4406

docs/api-reference/stages.rst

@@ -24,7 +24,7 @@ DeclarativeVersion
 
 .. autoclass:: pulpcore.plugin.stages.DeclarativeContent
    :no-members:
-   :members: pipeline_stages
+   :members: get_or_create_future
 
 
 .. _stages-api:
@@ -47,16 +47,12 @@ Artifact Related Stages
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 .. autoclass:: pulpcore.plugin.stages.ArtifactDownloader
-   :special-members: __call__
 
 .. autoclass:: pulpcore.plugin.stages.ArtifactSaver
-   :special-members: __call__
 
 .. autoclass:: pulpcore.plugin.stages.RemoteArtifactSaver
-   :special-members: __call__
 
 .. autoclass:: pulpcore.plugin.stages.QueryExistingArtifacts
-   :special-members: __call__
 
 
 .. _content-stages:
@@ -65,24 +61,21 @@ Content Related Stages
 ^^^^^^^^^^^^^^^^^^^^^^
 
 .. autoclass:: pulpcore.plugin.stages.ContentSaver
-   :special-members: __call__
    :private-members: _pre_save, _post_save
 
 .. autoclass:: pulpcore.plugin.stages.QueryExistingContents
-   :special-members: __call__
 
 .. autoclass:: pulpcore.plugin.stages.ResolveContentFutures
-   :special-members: __call__
 
 
 .. _content-association-stages:
 
 Content Association and Unassociation Stages
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+.. autoclass:: pulpcore.plugin.stages.RemoveDuplicates
+
 .. autoclass:: pulpcore.plugin.stages.ContentAssociation
-   :special-members: __call__
 
 .. autoclass:: pulpcore.plugin.stages.ContentUnassociation
-   :special-members: __call__
 

docs/plugin-writer/concepts/subclassing/serializers.rst

@@ -34,7 +34,7 @@ serializer needs to add that field as well.
           )
 
       class Meta:
-            fields = ContentSerializer.Meta.fields + ('relative_path',)
+            fields = SingleArtifactContentSerializer.Meta.fields + ('relative_path',)
             model = FileContent
 
 Help Text

docs/plugin-writer/concepts/tasks/add-remove.rst

@@ -3,7 +3,7 @@ Adding and Removing Content
 
 For adding and removing content, Pulp 3 provides a layered plugin API. The docs below explain our
 lower level API; this information is helpful to understand how a synchronize task works under the
-hood. **For implementation, it is highly recommended to use** :ref:`stages-docs` **instead**.
+hood.
 
 Repository Versions
 -------------------
@@ -33,6 +33,10 @@ working directory setup, and database cleanup after encountering failures.
 Synchronizing
 -------------
 
+.. tip::
+
+    Please consider using the high level :ref:`stages-docs` for actual implementations.
+
 Most plugins will define a synchronize task, which fetches content from a remote repository, and
 adds it to a Pulp repository.
 

pulpcore/plugin/stages/api.py

@@ -230,6 +230,11 @@ class EndStage(Stage):
     """
 
     async def __call__(self):
+        """
+        This method drains items from the last queue and drops them.
+
+        Importantly it does not try to put items into the nonexistent next queue.
+        """
         # We overwrite __call__ here to avoid trying to put None in `self._out_q`.
         async for _ in self.items():  # noqa
             pass

pulpcore/plugin/stages/association_stages.py

@@ -91,7 +91,8 @@ class RemoveDuplicates(Stage):
     """
     Stage allows plugins to remove content that would break repository uniqueness constraints.
 
-    This stage is expected to be added by the DeclarativeVersion. See that class for example usage.
+    This stage is expected to be added by the
+    :class:`~pulpcore.plugin.stages.DeclarativeVersion`. See that class for example usage.
     """
 
     def __init__(self, new_version, model, field_names):

pulpcore/plugin/stages/declarative_version.py

@@ -29,14 +29,31 @@ def __init__(self, first_stage, repository, mirror=True, download_artifacts=True
         The pipeline stages perform the following steps by default:
 
         1. Create the new :class:`~pulpcore.plugin.models.RepositoryVersion`
-        2. Query existing artifacts to determine which are already local to Pulp
-        3. Download any undownloaded :class:`~pulpcore.plugin.models.Artifact` objects.
-        4. Save the newly downloaded :class:`~pulpcore.plugin.models.Artifact` objects
-        5. Query for Content units already present in Pulp
-        6. Save new Content units not yet present in Pulp
-        7. Associate all content units with the new
-           :class:`~pulpcore.plugin.models.RepositoryVersion`.
-        8. Unassociate any content units not declared in the stream (only when sync_mode='mirror')
+        2. Use the provided `first_stage` to construct
+           :class:`~pulpcore.plugin.stages.DeclarativeContent`
+        3. Query existing artifacts to determine which are already local to Pulp with
+           :class:`~pulpcore.plugin.stages.QueryExistingArtifacts`
+        4. Download any undownloaded :class:`~pulpcore.plugin.models.Artifact` objects with
+           :class:`~pulpcore.plugin.stages.ArtifactDownloader`
+        5. Save the newly downloaded :class:`~pulpcore.plugin.models.Artifact` objects with
+           :class:`~pulpcore.plugin.stages.ArtifactSaver`
+        6. Query for Content units already present in Pulp with
+           :class:`~pulpcore.plugin.stages.QueryExistingContents`
+        7. Save new Content units not yet present in Pulp with
+           :class:`~pulpcore.plugin.stages.ContentSaver`
+        8. Attach :class:`~pulpcore.plugin.models.RemoteArtifact` to the
+           :class:`~pulpcore.plugin.models.Content` via
+           :class:`~pulpcore.plugin.stages.RemoteArtifactSaver`
+        9. Resolve the attached :class:`~asyncio.Future` of
+           :class:`~pulpcore.plugin.stages.DeclarativeContent` with
+           :class:`~pulpcore.plugin.stages.ResolveContentFutures`
+        10. Remove duplicate content in the repository version if `remove_duplicates` is given by
+            :class:`~pulpcore.plugin.stages.RemoveDuplicates`
+        11. Associate all content units with the new
+            :class:`~pulpcore.plugin.models.RepositoryVersion` with
+            :class:`~pulpcore.plugin.stages.ContentAssociation`
+        12. Unassociate any content units not declared in the stream (only when sync_mode='mirror')
+            with :class:`~pulpcore.plugin.stages.ContentUnassociation`
 
         To do this, the plugin writer should subclass the
         :class:`~pulpcore.plugin.stages.Stage` class and define its