Upload chunks to a local file system

As of now, every single chunk is going to be "uploaded" to a local file system instead of the defined default file storage.

In addition to that, the test test_delete_upload was refactored because it did not properly verify whether the committed upload was removed from the file system or not. It rather tested if the upload was deleted after sending an HTTP DELETE request.

closes #6253

CHANGES/6253.bugfix

@@ -0,0 +1 @@
+Made chunked uploads to be stored in a local file system instead of a default file storage

CHANGES/6253.feature

@@ -0,0 +1 @@
+Added the new setting CHUNKED_UPLOAD_DIR for configuring a default directory used for uploads

docs/workflows/upload-publish.rst

@@ -55,3 +55,9 @@ Putting this altogether, here is an example that uploads a 1.iso file in two chu
    http --form PUT :24817$UPLOAD file@./chunkab 'Content-Range:bytes 6291456-10485759/*'
    http --form PUT :24817$UPLOAD file@./chunkaa 'Content-Range:bytes 0-6291455/*'
    http POST :24817${UPLOAD}commit/ sha256=`sha256sum 1.iso | cut -d ' ' -f1`
+
+.. note::
+
+    Uploaded chunks are stored in a local file system. When the upload is committed, it is
+    automatically removed and a new artifact is created in a default file storage. The directory
+    which is used for temporary uploads is configurable via the setting ``CHUNKED_UPLOAD_DIR``.

pulpcore/app/migrations/0024_use_local_storage_for_uploads.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.11 on 2020-03-27 19:14
+
+import django.core.files.storage
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0023_change_exporter_models'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='upload',
+            name='file',
+            field=models.FileField(max_length=255, storage=django.core.files.storage.FileSystemStorage(location='/var/lib/pulp/upload/'), upload_to=''),
+        ),
+    ]

pulpcore/app/models/upload.py

@@ -1,7 +1,8 @@
 import hashlib
-import os
 
+from django.conf import settings
 from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
 from django.db import models
 from rest_framework import serializers
 
@@ -14,11 +15,12 @@ class Upload(BaseModel):
 
     Fields:
 
-        file (models.FileField): The stored file.
+        file (models.FileField): The uploaded file that is stored in a local file system.
         size (models.BigIntegerField): The size of the file in bytes.
     """
 
-    file = models.FileField(null=False, max_length=255)
+    file = models.FileField(null=False, max_length=255,
+                            storage=FileSystemStorage(location=settings.CHUNKED_UPLOAD_DIR))
     size = models.BigIntegerField()
 
     def append(self, chunk, offset, sha256=None):
@@ -30,7 +32,7 @@ def append(self, chunk, offset, sha256=None):
             offset (int): First byte position to write chunk to.
         """
         if not self.file:
-            self.file.save(os.path.join('upload', str(self.pk)), ContentFile(''))
+            self.file.save(str(self.pk), ContentFile(''))
 
         chunk_read = chunk.read()
         current_sha256 = hashlib.sha256(chunk_read).hexdigest()

pulpcore/app/settings.py

@@ -40,6 +40,7 @@
 
 FILE_UPLOAD_TEMP_DIR = os.path.join(MEDIA_ROOT, 'tmp/')
 WORKING_DIRECTORY = os.path.join(MEDIA_ROOT, 'tmp/')
+CHUNKED_UPLOAD_DIR = os.path.join(MEDIA_ROOT, 'upload/')
 
 # List of upload handler classes to be applied in order.
 FILE_UPLOAD_HANDLERS = ('pulpcore.app.files.HashingFileUploadHandler',)

pulpcore/app/viewsets/upload.py

@@ -41,7 +41,7 @@ class UploadViewSet(NamedModelViewSet,
     endpoint_name = 'uploads'
     queryset = Upload.objects.all()
     filterset_class = UploadFilter
-    http_method_names = ['get', 'post', 'head', 'put', 'delete']  # remove PATCH
+    http_method_names = ['get', 'post', 'head', 'put', 'delete']
 
     content_range_parameter = \
         Parameter(name='Content-Range', in_='header', required=True, type='string',