add new upload serializer

Required PR: pulp/pulpcore-plugin#122
closes #5464
https://pulp.plan.io/issues/5464

CHANGES/5464.feature

@@ -0,0 +1 @@
+Add upload functionality to the python contents endpoints.

pulp_python/app/serializers.py

@@ -1,14 +1,18 @@
 from gettext import gettext as _
+import os
+import shutil
+import tempfile
 
 from django.db import transaction
 from packaging import specifiers
 from rest_framework import serializers
 
 from pulpcore.plugin import models as core_models
-from pulpcore.plugin.models import Repository
 from pulpcore.plugin import serializers as core_serializers
 
 from pulp_python.app import models as python_models
+from pulp_python.app.tasks.upload import DIST_EXTENSIONS, DIST_TYPES
+from pulp_python.app.utils import parse_project_metadata
 
 
 class ClassifierSerializer(serializers.ModelSerializer):
@@ -69,7 +73,7 @@ class Meta:
         model = python_models.PythonDistribution
 
 
-class PythonPackageContentSerializer(core_serializers.SingleArtifactContentSerializer):
+class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploadSerializer):
     """
     A Serializer for PythonPackageContent.
     """
@@ -81,16 +85,20 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentSeria
     )
     packagetype = serializers.CharField(
         help_text=_('The type of the distribution package '
-                    '(e.g. sdist, bdist_wheel, bdist_egg, etc)')
+                    '(e.g. sdist, bdist_wheel, bdist_egg, etc)'),
+        read_only=True,
     )
     name = serializers.CharField(
-        help_text=_('The name of the python project.')
+        help_text=_('The name of the python project.'),
+        read_only=True,
     )
     version = serializers.CharField(
-        help_text=_('The packages version number.')
+        help_text=_('The packages version number.'),
+        read_only=True,
     )
     metadata_version = serializers.CharField(
-        help_text=_('Version of the file format')
+        help_text=_('Version of the file format'),
+        read_only=True,
     )
     summary = serializers.CharField(
         required=False, allow_blank=True,
@@ -179,6 +187,70 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentSeria
         many=True
     )
 
+    def deferred_validate(self, data):
+        """
+        Validate the rpm package data.
+
+        Args:
+            data (dict): Data to be validated
+
+        Returns:
+            dict: Data that has been validated
+
+        """
+        data = super().deferred_validate(data)
+
+        try:
+            filename = data['filename']
+        except KeyError:
+            raise serializers.ValidationError(detail={'filename': _('This field is required')})
+
+        if python_models.PythonPackageContent.objects.filter(filename=filename):
+            raise serializers.ValidationError(detail={'filename': _('This field must be unique')})
+
+        # iterate through extensions since splitext does not support things like .tar.gz
+        for ext, packagetype in DIST_EXTENSIONS.items():
+            if filename.endswith(ext):
+                # Copy file to a temp directory under the user provided filename, we do this
+                # because pkginfo validates that the filename has a valid extension before
+                # reading it
+                with tempfile.TemporaryDirectory() as td:
+                    temp_path = os.path.join(td, filename)
+                    artifact = data["artifact"]
+                    shutil.copy2(artifact.file.path, temp_path)
+                    metadata = DIST_TYPES[packagetype](temp_path)
+                    metadata.packagetype = packagetype
+                    break
+        else:
+            raise serializers.ValidationError(_(
+                "Extension on {} is not a valid python extension "
+                "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename)
+            )
+        _data = parse_project_metadata(vars(metadata))
+        _data['classifiers'] = [{'name': classifier} for classifier in metadata.classifiers]
+        _data['packagetype'] = metadata.packagetype
+        _data['version'] = metadata.version
+        _data['filename'] = filename
+        _data['relative_path'] = filename
+
+        data.update(_data)
+
+        new_content = python_models.PythonPackageContent.objects.filter(
+            filename=data['filename'],
+            packagetype=data['packagetype'],
+            name=data['classifiers'],
+            version=data['version']
+        )
+
+        if new_content.exists():
+            raise serializers.ValidationError(
+                _(
+                    "There is already a python package with relative path '{path}'."
+                ).format(path=data["relative_path"])
+            )
+
+        return data
+
     def create(self, validated_data):
         """
         Create a PythonPackageContent.
@@ -205,7 +277,7 @@ def create(self, validated_data):
         return package_content
 
     class Meta:
-        fields = core_serializers.SingleArtifactContentSerializer.Meta.fields + (
+        fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             'filename', 'packagetype', 'name', 'version', 'metadata_version', 'summary',
             'description', 'keywords', 'home_page', 'download_url', 'author', 'author_email',
             'maintainer', 'maintainer_email', 'license', 'requires_python', 'project_url',
@@ -215,30 +287,13 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
-class PythonOneShotUploadSerializer(serializers.Serializer):
-    """
-    A Serializer for PythonOneShotUpload.
-    """
-
-    repository = serializers.HyperlinkedRelatedField(
-        help_text=_('A URI of the repository.'),
-        required=False,
-        queryset=Repository.objects.all(),
-        view_name='repositories-detail',
-    )
-    file = serializers.FileField(
-        help_text=_("The python file (i.e. .whl or .tar.gz)."),
-        required=True,
-    )
-
-
 class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer):
     """
     A Serializer for PythonPackageContent.
     """
 
     class Meta:
-        fields = core_serializers.SingleArtifactContentSerializer.Meta.fields + (
+        fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             'filename', 'packagetype', 'name', 'version',
         )
         model = python_models.PythonPackageContent

pulp_python/app/viewsets.py

@@ -1,12 +1,8 @@
-from gettext import gettext as _
-
-from django.db.utils import IntegrityError
 from drf_yasg.utils import swagger_auto_schema
-from rest_framework import serializers, viewsets
 from rest_framework.decorators import action
 
 from pulpcore.plugin import viewsets as platform
-from pulpcore.plugin.models import Artifact, RepositoryVersion
+from pulpcore.plugin.models import RepositoryVersion
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
     RepositorySyncURLSerializer,
@@ -16,7 +12,6 @@
 from pulp_python.app import models as python_models
 from pulp_python.app import serializers as python_serializers
 from pulp_python.app import tasks
-from pulp_python.app.tasks.upload import one_shot_upload
 
 
 class PythonDistributionViewSet(platform.BaseDistributionViewSet):
@@ -50,7 +45,7 @@ class Meta:
         }
 
 
-class PythonPackageContentViewSet(platform.ContentViewSet):
+class PythonPackageSingleArtifactContentUploadViewSet(platform.SingleArtifactContentUploadViewSet):
     """
     <!-- User-facing documentation, rendered as html-->
     PythonPackageContent represents each individually installable Python package. In the Python
@@ -68,61 +63,6 @@ class PythonPackageContentViewSet(platform.ContentViewSet):
     filterset_class = PythonPackageContentFilter
 
 
-class PythonOneShotUploadViewSet(viewsets.ViewSet):
-    """
-    ViewSet for OneShotUpload
-    """
-
-    endpoint_name = 'upload'
-    serializer_class = python_serializers.PythonOneShotUploadSerializer
-
-    def create(self, request):
-        """
-        <!-- User-facing documentation, rendered as html-->
-        This endpoint is part of the <a href="workflows/upload.html">Upload workflow.</a> Create
-        a PythonPackageContent here by specifying an uploaded Artifact. `pulp-python` will inspect
-        parse the metadata directly from the file.
-
-        """
-        try:
-            artifact = Artifact.init_and_validate(request.data['file'])
-        except KeyError:
-            raise serializers.ValidationError(detail={'_artifact': _('This field is required')})
-
-        try:
-            filename = request.data['filename']
-        except KeyError:
-            raise serializers.ValidationError(detail={'filename': _('This field is required')})
-
-        if python_models.PythonPackageContent.objects.filter(filename=filename):
-            raise serializers.ValidationError(detail={'filename': _('This field must be unique')})
-
-        if 'repository' in request.data:
-            serializer = python_serializers.PythonOneShotUploadSerializer(
-                data=request.data, context={'request': request})
-            serializer.is_valid(raise_exception=True)
-            repository = serializer.validated_data['repository']
-            repository_pk = repository.pk
-        else:
-            repository_pk = None
-
-        try:
-            artifact.save()
-        except IntegrityError:
-            artifact = Artifact.objects.get(sha256=artifact.sha256)
-
-        result = enqueue_with_reservation(
-            one_shot_upload,
-            [artifact],
-            kwargs={
-                'artifact_pk': artifact.pk,
-                'filename': filename,
-                'repository_pk': repository_pk,
-            }
-        )
-        return platform.OperationPostponedResponse(result, request)
-
-
 class PythonRemoteFilter(platform.RemoteFilter):
     """
     FilterSet for PythonRemote.

pulp_python/tests/functional/api/test_crud_content_unit.py

@@ -6,7 +6,7 @@
 from pulp_smash.pulp3.constants import REPO_PATH
 from pulp_smash.pulp3.utils import delete_orphans
 
-from pulp_python.tests.functional.constants import (PYTHON_CONTENT_PATH, PYTHON_UPLOAD_PATH,
+from pulp_python.tests.functional.constants import (PYTHON_CONTENT_PATH,
                                                     PYTHON_WHEEL_URL, PYTHON_WHEEL_FILENAME)
 from pulp_python.tests.functional.utils import skip_if
 from pulp_python.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
@@ -41,9 +41,11 @@ def test_01_upload_file_without_repo(self):
         3) check created resource of completed task
         4) ensure only one and it's a content unit
         """
-        task_url = self.client.post(PYTHON_UPLOAD_PATH,
-                                    files=self.test_file,
-                                    data={'filename': PYTHON_WHEEL_FILENAME})
+        task_url = self.client.post(
+            PYTHON_CONTENT_PATH,
+            files=self.test_file,
+            data={'filename': PYTHON_WHEEL_FILENAME,
+                  'relative_path': PYTHON_WHEEL_FILENAME})
         task = self.client.get(task_url['task'])
         created_resource = task['created_resources'][0]
         content_unit = self.client.get(created_resource)
@@ -61,9 +63,10 @@ def test_02_upload_file_with_repo(self):
         """
         repo = self.client.post(REPO_PATH, data={'name': 'foo'})
         self.addCleanup(self.client.delete, repo['_href'])
-        task_url = self.client.post(PYTHON_UPLOAD_PATH,
+        task_url = self.client.post(PYTHON_CONTENT_PATH,
                                     files=self.test_file,
                                     data={'filename': PYTHON_WHEEL_FILENAME,
+                                          'relative_path': PYTHON_WHEEL_FILENAME,
                                           'repository': repo['_href']})
         task = self.client.get(task_url['task'])
         new_repo_version = task['created_resources'][0]
@@ -86,19 +89,21 @@ def test_03_upload_duplicate_file_without_repo(self):
         2) upload the same file again
         3) this should fail/send an error
         """
-        task_url = self.client.post(PYTHON_UPLOAD_PATH,
+        task_url = self.client.post(PYTHON_CONTENT_PATH,
                                     files=self.test_file,
-                                    data={'filename': PYTHON_WHEEL_FILENAME})
+                                    data={'filename': PYTHON_WHEEL_FILENAME,
+                                          'relative_path': PYTHON_WHEEL_FILENAME})
         task = self.client.get(task_url['task'])
         created_resource = task['created_resources'][0]
         content_unit = self.client.get(created_resource)
         new_filename = content_unit['filename']
         self.assertEqual(new_filename, PYTHON_WHEEL_FILENAME)
         self.assertEqual(len(task['created_resources']), 1)
         try:
-            self.client.post(PYTHON_UPLOAD_PATH,
+            self.client.post(PYTHON_CONTENT_PATH,
                              files=self.test_file,
-                             data={'filename': PYTHON_WHEEL_FILENAME})
+                             data={'filename': PYTHON_WHEEL_FILENAME,
+                                   'relative_path': PYTHON_WHEEL_FILENAME})
         except Exception as e:
             self.assertEqual(e.response.status_code, 400)
 
@@ -123,9 +128,10 @@ def setUpClass(cls):
         delete_orphans(cls.cfg)
         cls.client = api.Client(cls.cfg, api.json_handler)
         cls.test_file = {'file': utils.http_get(PYTHON_WHEEL_URL)}
-        task_url = cls.client.post(PYTHON_UPLOAD_PATH,
+        task_url = cls.client.post(PYTHON_CONTENT_PATH,
                                    files=cls.test_file,
-                                   data={'filename': PYTHON_WHEEL_FILENAME})
+                                   data={'filename': PYTHON_WHEEL_FILENAME,
+                                         'relative_path': PYTHON_WHEEL_FILENAME})
         task = cls.client.get(task_url['task'])
         created_resource = task['created_resources'][0]
         cls.content_unit = cls.client.get(created_resource)

pulp_python/tests/functional/constants.py

@@ -3,7 +3,6 @@
 from pulp_smash.constants import PULP_FIXTURES_BASE_URL
 from pulp_smash.pulp3.constants import (
     BASE_DISTRIBUTION_PATH,
-    BASE_PATH,
     BASE_PUBLICATION_PATH,
     BASE_REMOTE_PATH,
     CONTENT_PATH
@@ -24,8 +23,6 @@
 
 PYTHON_REMOTE_PATH = urljoin(BASE_REMOTE_PATH, 'python/python/')
 
-PYTHON_UPLOAD_PATH = urljoin(BASE_PATH, 'python/upload/')
-
 
 # Specifier for testing empty syncs, or no excludes
 PYTHON_EMPTY_PROJECT_SPECIFIER = []