Add support for Amazon S3

fixes #3900
https://pulp.plan.io/issues/3900

docs/installation/configuration.rst

@@ -42,6 +42,24 @@ DATABASES
    `Django documentation on databases <https://docs.djangoproject.com/en/2
    .1/ref/settings/#databases>`_
 
+DEFAULT_FILE_STORAGE
+^^^^^^^^^^^^^^^^^^^^
+
+   By default, Pulp uses the local filesystem to store files. The default option which
+   uses the local filesystem is ``pulpcore.app.models.storage.FileSystem``.
+
+   This can be configured though to alternatively use `Amazon S3 <https://aws.amazon.com/s3/>`_. To
+   use S3, set ``DEFAULT_FILE_STORAGE`` to ``storages.backends.s3boto3.S3Boto3Storage``. For more
+   information about different Pulp storage options, see the `storage documentation <storage>`_.
+
+MEDIA_ROOT
+^^^^^^^^^^
+
+   The location where Pulp will store files. By default this is `/var/lib/pulp/`.
+
+   If you're using S3, point this to the path in your bucket you want to save files. See the
+   `storage documentation <storage>`_ for more info.
+
 LOGGING
 ^^^^^^^
 

docs/installation/index.rst

@@ -6,6 +6,7 @@ Installation
 
    instructions
    configuration
+   storage
    distributed-installation
    migration
 

docs/installation/storage.rst

@@ -0,0 +1,46 @@
+Storage
+=======
+
+.. _storage:
+
+-----------
+
+  Pulp uses `django-storages <https://django-storages.readthedocs.io/>`_ to support multiple storage
+  backends. If no backend is configured, Pulp will by default use the local filesystem. If you want
+  to use another storage backend such as Amazon Simple Storage Service (S3), you'll need to
+  configure Pulp.
+
+Amazon S3
+^^^^^^^^^
+
+Setting up S3
+-------------
+
+  In order to use Amazon S3, you'll need to set up an AWS account. Then you'll need to create a
+  bucket for Pulp to use. Then you'll need to go into Identity and Access Management (IAM) in AWS to
+  create a user that Pulp will use to access your S3 bucket. Save the access key id and secret
+  access key.
+
+Configuring Pulp
+----------------
+
+  To have Pulp use S3, you'll need to install the optional django-stroages Python package::
+
+      pip install django-storages
+
+  Next you'll need to set ``DEFAULT_FILE_STORAGE`` to ``storages.backends.s3boto3.S3Boto3Storage``
+  in your Pulp settings. At a minimum, you'll also need to set ``AWS_ACCESS_KEY_ID``,
+  ``AWS_SECRET_ACCESS_KEY``, and ``AWS_STORAGE_BUCKET_NAME``. For more S3 configuration options, see
+  the `django-storages documents <https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html>`_.
+
+  You will also want to set the ``MEDIA_ROOT`` configuration option. This will be the path in your
+  bucket that Pulp will use. An empty string is acceptable as well if you want Pulp to create its
+  folders in the top level of the bucket.
+
+  Here is an example configuration that will use a bucket called ``pulp3``::
+
+        AWS_ACCESS_KEY_ID = 'AKIAIT2Z5TDYPX3ARJBA'
+        AWS_SECRET_ACCESS_KEY = 'qR+vjWPU50fCqQuUWbj9Fain/j2pV+ZtBCiDiieS'
+        AWS_STORAGE_BUCKET_NAME = 'pulp3'
+        DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+        MEDIA_ROOT = ''

pulpcore/app/models/fields.py

@@ -35,7 +35,11 @@ def pre_save(self, model_instance, add):
                                'Artifact storage. Files must be stored outside this location '
                                'prior to Artifact creation.'))
         file = super().pre_save(model_instance, add)
-        if file and file._committed and add:
+        if file and file._committed and add and str(file) != upload_to:
+            # the file needs to be moved into place
             file._file = TemporaryDownloadedFile(open(file.name, 'rb'))
             file._committed = False
-        return super().pre_save(model_instance, add)
+            return super().pre_save(model_instance, add)
+        else:
+            # the file is already in place so just return it
+            return file

pulpcore/app/settings.py

@@ -78,7 +78,8 @@
 # Optional apps that help with development, or augment Pulp in some non-critical way
 OPTIONAL_APPS = [
     'crispy_forms',
-    'django_extensions'
+    'django_extensions',
+    'storages',
 ]
 
 for app in OPTIONAL_APPS:

pulpcore/content/handler.py

@@ -10,7 +10,8 @@
 
 from aiohttp.client_exceptions import ClientResponseError
 from aiohttp.web import FileResponse, StreamResponse
-from aiohttp.web_exceptions import HTTPForbidden, HTTPNotFound
+from aiohttp.web_exceptions import HTTPForbidden, HTTPFound, HTTPNotFound
+from django.conf import settings
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.db import IntegrityError, transaction
 from pulpcore.app.models import Artifact, ContentArtifact, Distribution, Remote
@@ -176,7 +177,7 @@ async def _match_and_stream(self, path, request):
             pass
         else:
             if ca.artifact:
-                return FileResponse(ca.artifact.file.name)
+                return self._handle_file_response(ca.artifact.file)
             else:
                 return await self._stream_content_artifact(request, StreamResponse(), ca)
 
@@ -186,7 +187,7 @@ async def _match_and_stream(self, path, request):
         except ObjectDoesNotExist:
             pass
         else:
-            return FileResponse(pm.file.name)
+            return self.handle_file_response(pm.file)
 
         # pass-through
         if publication.pass_through:
@@ -207,7 +208,7 @@ async def _match_and_stream(self, path, request):
                 pass
             else:
                 if ca.artifact:
-                    return FileResponse(ca.artifact.file.name)
+                    return self._handle_file_response(ca.artifact.file)
                 else:
                     return await self._stream_content_artifact(request, StreamResponse(), ca)
         raise PathNotResolved(path)
@@ -303,3 +304,27 @@ def _save_content_artifact(self, download_result, content_artifact):
             content_artifact.artifact = artifact
             content_artifact.save()
         return artifact
+
+    def _handle_file_response(self, file):
+        """
+        Handle response for file.
+
+        Depending on where the file storage (e.g. filesystem, S3, etc) this could be responding with
+        the file (filesystem) or a redirect (S3).
+
+        Args:
+            file (:class:`django.db.models.fields.files.FieldFile`): File to respond with
+
+        Raises:
+            :class:`aiohttp.web_exceptions.HTTPFound`: When we need to redirect to the file
+            NotImplementedError: If file is stored in a file storage we can't handle
+
+        Returns:
+            The :class:`aiohttp.web.FileResponse` for the file.
+        """
+        if settings.DEFAULT_FILE_STORAGE == 'pulpcore.app.models.storage.FileSystem':
+            return FileResponse(file.name)
+        elif settings.DEFAULT_FILE_STORAGE == 'storages.backends.s3boto3.S3Boto3Storage':
+            raise HTTPFound(file.url)
+        else:
+            raise NotImplementedError()