Add download and changeset.

closes #2168

docs/contributing/plugin_api/changeset.rst

@@ -0,0 +1,47 @@
+pulpcore.plugin.changeset
+=========================
+
+All classes documented here should be imported directly from
+the ``pulpcore.plugin.changeset`` namespace.
+
+.. automodule:: pulpcore.plugin.changeset
+
+.. autoclass:: pulpcore.plugin.changeset.ChangeSet
+    :members: apply
+
+
+New Content & Artifacts
+-----------------------
+
+Classes used to define *new* content to be added to a repository.
+
+
+.. autoclass:: pulpcore.plugin.changeset.RemoteContent
+    :members: artifacts
+
+.. autoclass:: pulpcore.plugin.changeset.RemoteArtifact
+    :members: content
+
+
+Reporting
+---------
+
+Reports and Exceptions.
+
+
+.. autoclass:: pulpcore.plugin.changeset.ChangeReport
+    :members:
+
+.. autoclass:: pulpcore.plugin.changeset.ChangeFailed
+    :show-inheritance:
+    :members:
+
+
+Additional Tools
+----------------
+
+.. autoclass:: pulpcore.plugin.changeset.BatchIterator
+    :special-members: __len__, __iter__
+
+.. autoclass:: pulpcore.plugin.changeset.SizedIterable
+    :special-members: __len__

docs/contributing/plugin_api/download.rst

@@ -0,0 +1,97 @@
+pulpcore.plugin.download
+========================
+
+All classes documented here should be imported directly from
+the ``pulpcore.plugin.download`` namespace.
+
+.. automodule:: pulpcore.plugin.download
+
+
+
+Single File
+-----------
+
+
+.. autoclass:: pulpcore.plugin.download.HttpDownload
+    :members:
+    :special-members: __call__
+
+.. autoclass:: pulpcore.plugin.download.FileDownload
+    :members:
+    :special-members: __call__
+
+
+Multiple Files (concurrent)
+---------------------------
+
+
+.. autoclass:: pulpcore.plugin.download.Batch
+    :members: download, shutdown
+    :special-members: __call__
+
+
+File Validation
+---------------
+
+.. autoclass:: pulpcore.plugin.download.SizeValidation
+    :members:
+
+.. autoclass:: pulpcore.plugin.download.DigestValidation
+    :members:
+    :special-members: __call__
+
+
+Writers
+-------
+
+Downloading consists of two related operations.  First, is reading the file content
+from a remote source.  Second, is writing those bits locally.  The most typical case is
+to write the bits to a file on the local filesystem and is accomplished using
+a `FileWriter`.  Another case, is to store the file content (text) in memory and then
+inspect as a `str`.  As a convenience, this may be done using the `BufferWriter`.
+
+
+.. autoclass:: pulpcore.plugin.download.FileWriter
+    :members:
+
+.. autoclass:: pulpcore.plugin.download.BufferWriter
+    :members:
+
+
+Settings
+--------
+
+.. autoclass:: pulpcore.download.SSL
+    :members:
+
+.. autoclass:: pulpcore.download.User
+    :members:
+
+.. autoclass:: pulpcore.download.Timeout
+    :members:
+
+
+Errors
+------
+
+A download is successful unless an exception is raised.
+
+.. autoclass:: pulpcore.plugin.download.DownloadError
+    :show-inheritance:
+    :members: download, reason
+
+.. autoclass:: pulpcore.plugin.download.DownloadFailed
+    :show-inheritance:
+    :members: download, reason
+
+.. autoclass:: pulpcore.plugin.download.NotAuthorized
+    :show-inheritance:
+    :members: download, reason
+
+.. autoclass:: pulpcore.plugin.download.NotFound
+    :show-inheritance:
+    :members: download, reason
+
+.. autoclass:: pulpcore.plugin.download.ValidationError
+    :show-inheritance:
+    :members: download, reason

docs/contributing/plugin_api/index.rst

@@ -11,6 +11,8 @@ Plugin API reaches stability with v1.0. For the latest version of the Plugin API
     models
     serializers
     viewsets
+    changeset
+    download
 
 .. automodule:: pulpcore.plugin
     :imported-members:

platform/pulpcore/app/models/catalog.py

@@ -34,6 +34,9 @@ class DownloadCatalog(Model):
     artifact = models.ForeignKey(Artifact, on_delete=models.CASCADE)
     importer = models.ForeignKey(Importer, on_delete=models.CASCADE)
 
+    class Meta:
+        unique_together = ('artifact', 'importer')
+
     def __str__(self):
         """
         Human-readable representation of this model.
@@ -42,5 +45,8 @@ def __str__(self):
         interface.
         """
 
-        return _('{artifact} is retrievable at {url} by {importer}'.format(
-            artifact=self.artifact, url=self.url, importer=self.importer))
+        return _(
+            '{artifact} is retrievable at {url} by {importer}'.format(
+                artifact=self.artifact,
+                url=self.url,
+                importer=self.importer))

platform/pulpcore/download/__init__.py

@@ -0,0 +1,71 @@
+"""
+This package provides objects used for downloading files.
+The workhorse is the ``Download`` object which performs the task of downloading a single file.
+A more natural name for this class might be a download `Task` but this might cause confusion
+with celery tasks in Pulp.  Here, a `download` is used as a noun and has a command-object
+design pattern.  That is, the download is callable.  The Batch provides a way to perform multiple
+downloads concurrently.
+
+The model::
+
+                     |*-------1 Batch
+            Download |--------------------------------------------------|
+               ^     |1-----------------* Validation *-----------------1| Writer
+               |                               ^                            ^
+        -------|------                         |                            |
+        |      |      |             ---------------------           -----------------
+        |      |      |             |                    |          |               |
+  HttpDownload |  FtpDownload  SizeValidation  DigestValidation  FileWriter    BufferWriter
+               |
+         FileDownload
+
+Recipes:
+
+    A single download.
+
+    >>>
+    >>> download = HttpDownload('http://my-url', FileWriter('put-file-here'))
+    >>> download()
+    >>> # Go use the file.
+    >>>
+
+    Multiple downloads concurrently.
+
+    >>>
+    >>> downloads = [
+    >>>     HttpDownload('http://my-url0', FileWriter('put-file-here0')),
+    >>>     FileDownload('file://my-url1', FileWriter('put-file-here1')),
+    >>>     FtpDownload('ftp://my-url2', FileWriter('put-file-here2')),
+    >>> ]
+    >>>
+    >>> with Batch(downloads) as batch:
+    >>>     for plan in batch():
+    >>>         try:
+    >>>             plan.result()
+    >>>         except Exception:
+    >>>             # Failed
+    >>>         else:
+    >>>             # Use the downloaded file \o/
+    >>>
+
+    Download a text file to a use as a string.
+
+    >>>
+    >>> download = HttpDownload('http://my-url', BufferWriter())
+    >>> download()
+    >>> document = download.writer.read()
+    >>> # Use the document
+    >>>
+
+"""
+
+from .batch import Batch  # noqa
+from .delegation import delegate  # noqa
+from .error import DownloadError, DownloadFailed, NotFound, NotAuthorized  # noqa
+from .ftp import FtpDownload  # noqa
+from .file import FileDownload  # noqa
+from .http import HttpDownload  # noqa
+from .settings import Settings, SSL, Timeout, User  # noqa
+from .single import Download  # noqa
+from .validation import ValidationError, SizeValidation, DigestValidation  # noqa
+from .writer import Writer, FileWriter, BufferWriter  # noqa