WIP one-shot upload works, adding tests/docs

fixes #4396
https://pulp.plan.io/issues/4396

pulp_python/app/serializers.py

@@ -4,6 +4,7 @@
 from packaging import specifiers
 from rest_framework import serializers
 
+from pulpcore.plugin.models import Repository
 from pulpcore.plugin import serializers as core_serializers
 
 from pulp_python.app import models as python_models
@@ -222,7 +223,7 @@ class PythonOneShotUploadSerializer(serializers.Serializer):
         help_text=_('A URI of the repository.'),
         required=False,
         queryset=Repository.objects.all(),
-        view_name='respositories-detail',
+        view_name='repositories-detail',
     )
     file = serializers.FileField(
         help_text=_("The python file."),

pulp_python/app/tasks/upload.py

@@ -1,29 +1,84 @@
-from pulpcore.app.models.content import Artifact, ContentArtifact
+import os
+from gettext import gettext as _
+import pkginfo
+import shutil
+import tempfile
+
+from pulpcore.plugin.models import Artifact, CreatedResource, Repository, RepositoryVersion
+from rest_framework import serializers
+
 from pulp_python.app.models import PythonPackageContent
+from pulp_python.app.utils import parse_project_metadata
+
+import logging
+log=logging.getLogger('moose')
+
+DIST_EXTENSIONS = {
+    ".whl": "bdist_wheel",
+    ".exe": "bdist_wininst",
+    ".egg": "bdist_egg",
+    ".tar.bz2": "sdist",
+    ".tar.gz": "sdist",
+    ".zip": "sdist",
+}
+
+DIST_TYPES = {
+    "bdist_wheel": pkginfo.Wheel,
+    "bdist_wininst": pkginfo.Distribution,
+    "bdist_egg": pkginfo.BDist,
+    "sdist": pkginfo.SDist,
+}
 
-def one_shot_upload(artifact, filename, data, repository=None):
+
+def one_shot_upload(artifact_pk, filename, repository_pk=None):
     """
     One shot upload for pulp_python
 
     Args:
-        artifact: validated artifact
+        artifact_pk: validated artifact
         filename: file name
-        repository: optional repository to add Content to
+        repository_pk: optional repository to add Content to
     """
 
+    # iterate through extensions since splitext does not support things like .tar.gz
+    for ext, packagetype in DIST_EXTENSIONS.items():
+        if filename.endswith(ext):
+            # Copy file to a temp directory under the user provided filename, we do this
+            # because pkginfo validates that the filename has a valid extension before
+            # reading it
+            with tempfile.TemporaryDirectory() as td:
+                temp_path = os.path.join(td, filename)
+                artifact = Artifact.objects.get(pk=artifact_pk)
+                shutil.copy2(artifact.file.path, temp_path)
+                metadata = DIST_TYPES[packagetype](temp_path)
+                metadata.packagetype = packagetype
+                break
+    else:
+        raise serializers.ValidationError(_(
+            "Extension on {} is not a valid python extension "
+            "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename)
+                                          )
+    data = parse_project_metadata(vars(metadata))
+    data['classifiers'] = [{'name': classifier} for classifier in metadata.classifiers]
+    data['packagetype'] = metadata.packagetype
+    data['version'] = metadata.version
+    data['filename'] = filename
+    data['_relative_path'] = filename
+
     new_content = PythonPackageContent.objects.create(
         filename=filename,
-        package_type=data['package_type'],
-        name=data['classifier'],
+        packagetype=metadata.packagetype,
+        name=data['classifiers'],
         version=data['version']
     )
 
-    new_content_artifact = ContentArtifact.objects.create(
-        artifact=artifact,
-        content = new_content,
-        relative_path=filename
-    )
+    log.warn(new_content)
+    queryset = PythonPackageContent.objects.filter(pk=new_content.pk)
 
-    if repository:
+    if repository_pk:
+        repository = Repository.objects.get(pk=repository_pk)
         with RepositoryVersion.create(repository) as new_version:
-            new_version.add_content(new_content_artifact)
+            new_version.add_content(queryset)
+
+    resource = CreatedResource(content_object=new_content)
+    resource.save()

pulp_python/app/urls.py

@@ -0,0 +1,8 @@
+from django.conf.urls import url
+
+from .viewsets import PythonOneShotUploadViewSet
+
+
+urlpatterns = [
+    url(r'python/upload/$', PythonOneShotUploadViewSet.as_view({'post': 'create'}))
+]

pulp_python/app/viewsets.py

@@ -1,17 +1,12 @@
-import os
 from gettext import gettext as _
-import pkginfo
-import shutil
-import tempfile
 
-from django.db import transaction
+from django.db.utils import IntegrityError
 from drf_yasg.utils import swagger_auto_schema
-from rest_framework import status, serializers
+from rest_framework import status, serializers, viewsets
 from rest_framework.decorators import action
-from rest_framework.response import Response
 
 from pulpcore.plugin import viewsets as platform
-from pulpcore.plugin.models import Artifact, RepositoryVersion
+from pulpcore.plugin.models import Artifact, Repository, RepositoryVersion
 from pulpcore.plugin.serializers import (
     AsyncOperationResponseSerializer,
     RepositorySyncURLSerializer,
@@ -21,23 +16,7 @@
 from pulp_python.app import models as python_models
 from pulp_python.app import serializers as python_serializers
 from pulp_python.app import tasks
-from pulp_python.app.utils import parse_project_metadata
-
-DIST_EXTENSIONS = {
-    ".whl": "bdist_wheel",
-    ".exe": "bdist_wininst",
-    ".egg": "bdist_egg",
-    ".tar.bz2": "sdist",
-    ".tar.gz": "sdist",
-    ".zip": "sdist",
-}
-
-DIST_TYPES = {
-    "bdist_wheel": pkginfo.Wheel,
-    "bdist_wininst": pkginfo.Distribution,
-    "bdist_egg": pkginfo.BDist,
-    "sdist": pkginfo.SDist,
-}
+from pulp_python.app.tasks.upload import one_shot_upload
 
 
 class PythonDistributionViewSet(platform.BaseDistributionViewSet):
@@ -89,15 +68,14 @@ class PythonPackageContentViewSet(platform.ContentViewSet):
     filterset_class = PythonPackageContentFilter
 
 
-class PythonOneShotUploadViewSet(platform.ViewSet):
+class PythonOneShotUploadViewSet(viewsets.ViewSet):
     """
     ViewSet for OneShotUpload
     """
 
     endpoint_name = 'upload'
     serializer_class = python_serializers.PythonOneShotUploadSerializer
 
-    @transaction.atomic
     def create(self, request):
         """
         <!-- User-facing documentation, rendered as html-->
@@ -107,55 +85,37 @@ def create(self, request):
 
         """
         try:
-            artifact = self.get_resource(request.data['_artifact'], Artifact)
+            artifact = Artifact.init_and_validate(request.data['file'])
         except KeyError:
             raise serializers.ValidationError(detail={'_artifact': _('This field is required')})
 
         try:
             filename = request.data['filename']
         except KeyError:
             raise serializers.ValidationError(detail={'filename': _('This field is required')})
-
-        # iterate through extensions since splitext does not support things like .tar.gz
-        for ext, packagetype in DIST_EXTENSIONS.items():
-            if filename.endswith(ext):
-
-                # Copy file to a temp directory under the user provided filename, we do this
-                # because pkginfo validates that the filename has a valid extension before
-                # reading it
-                with tempfile.TemporaryDirectory() as td:
-                    temp_path = os.path.join(td, filename)
-                    shutil.copy2(artifact.file.path, temp_path)
-                    metadata = DIST_TYPES[packagetype](temp_path)
-                    metadata.packagetype = packagetype
-                    break
+
+        if 'repository' in request.data:
+            serializer = python_serializers.PythonOneShotUploadSerializer(
+                data=request.data, context={'request': request})
+            serializer.is_valid(raise_exception=True)
+            repository = serializer.validated_data['repository']
+            repository_pk = repository.pk
         else:
-            raise serializers.ValidationError(_(
-                "Extension on {} is not a valid python extension "
-                "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename)
-            )
-
-        data = parse_project_metadata(vars(metadata))
-        data['classifiers'] = [{'name': classifier} for classifier in metadata.classifiers]
-        data['packagetype'] = metadata.packagetype
-        data['version'] = metadata.version
-        data['filename'] = filename
-        data['_artifact'] = request.data['_artifact']
-        data['_relative_path'] = filename
-
-        serializer = self.get_serializer(data=data)
-        serializer.is_valid(raise_exception=True)
-        repository = serializer.validated_data['repository']
-        serializer.save()
+            repository_pk = None
+
+        try:
+            artifact.save()
+        except IntegrityError:
+            artifact = Artifact.objects.get(sha256=artifact.sha256)
+
 
         result = enqueue_with_reservation(
-            tasks.upload,
+            one_shot_upload,
             [artifact],
             kwargs={
-                'artifact': artifact,
+                'artifact_pk': artifact.pk,
                 'filename': filename,
-                'data': data,
-                'repository': repository,
+                'repository_pk': repository_pk,
             }
         )
         return platform.OperationPostponedResponse(result, request)