Add one-shot upload

Adds one-shot upload feature, optionally specifying a repo
Updates tests and docs accordingly

fixes #4396
https://pulp.plan.io/issues/4396

CHANGES/4396.feature

@@ -0,0 +1,2 @@
+Users can upload a file to create content and optionally add to a repo in one step known as
+one-shot upload

docs/_scripts/base.sh

@@ -1,13 +1,34 @@
-export BASE_ADDR=http://localhost:24817
-export CONTENT_ADDR=http://localhost:24816
+#!/usr/bin/env bash
 
-wait_for_pulp() {
-  unset CREATED_RESOURCE
-  local task_url=$1
-  while [ -z "$CREATED_RESOURCE" ]
+echo "Setting environment variables for default hostname/port for the API and the Content app"
+export BASE_ADDR=${BASE_ADDR:-http://localhost:24817}
+export CONTENT_ADDR=${CONTENT_ADDR:-http://localhost:24816}
 
-  do
-    sleep 1
-    export CREATED_RESOURCE=$(http $BASE_ADDR$task_url | jq -r '.created_resources | first')
-  done
+# Necessary for `django-admin`
+export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
+
+# Poll a Pulp task until it is finished.
+wait_until_task_finished() {
+    echo "Polling the task until it has reached a final state."
+    local task_url=$1
+    while true
+    do
+        local response=$(http $task_url)
+        local state=$(jq -r .state <<< ${response})
+        jq . <<< "${response}"
+        case ${state} in
+            failed|canceled)
+                echo "Task in final state: ${state}"
+                exit 1
+                ;;
+            completed)
+                echo "$task_url complete."
+                break
+                ;;
+            *)
+                echo "Still waiting..."
+                sleep 1
+                ;;
+        esac
+    done
 }

docs/_scripts/upload.sh

@@ -0,0 +1,17 @@
+pclean
+prestart
+
+source ./base.sh
+
+# Upload your file, optionally specifying a repository
+export TASK_URL=$(http --form POST $BASE_ADDR/pulp/api/v3/python/upload/ file@../../shelf_reader-0.1-py2-none-any.whl filename=shelf_reader-0.1-py2-none-any.whl | \
+	jq -r '.task')
+
+wait_until_task_finished $BASE_ADDR$TASK_URL
+
+# If you want to copy/paste your way through the guide,
+# create an environment variable for the repository URI.
+export CONTENT_HREF=$(http $BASE_ADDR$TASK_URL | jq -r '.created_resources | first')
+
+# Let's inspect our newly created content.
+http $BASE_ADDR$CONTENT_HREF

docs/_scripts/upload_with_repo.sh

@@ -0,0 +1,20 @@
+pclean
+prestart
+
+source base.sh
+
+source repo.sh
+
+#Upload your file, optionally specifying a repository
+export TASK_URL=$(http --form POST $BASE_ADDR/pulp/api/v3/python/upload/ file@../../shelf_reader-0.1-py2-none-any.whl filename=shelf_reader-0.1-py2-none-any.whl repository=$REPO_HREF | \
+        jq -r '.task')
+
+wait_until_task_finished $BASE_ADDR$TASK_URL
+
+# If you want to copy/paste your way through the guide,
+# create an environment variable for the repository URI.
+export CONTENT_HREF=$(http $BASE_ADDR$TASK_URL | \
+        jq -r '.created_resources | first')
+
+#Let's inspect our newly created content.
+http $BASE_ADDR$CONTENT_HREF

docs/workflows/upload.rst

@@ -1,49 +1,84 @@
 Upload Content
 ==============
 
-Upload a file to Pulp
----------------------
+One-shot upload a file to Pulp
+------------------------------
 
-Each artifact in Pulp represents a file. They can be created during sync or created manually by uploading a file::
+Each artifact in Pulp represents a file. They can be created during sync or created manually by uploading a file via
+one-shot upload. One-shot upload takes a file you specify, creates an artifact, and creates content from that artifact.
+The python plugin will inspect the file and populate its metadata.
 
-    $ export ARTIFACT_HREF=$(http --form POST $BASE_ADDR/pulp/api/v3/artifacts/ file@./shelf_reader-0.1-py2-none-any.whl | jq -r '._href')
+.. literalinclude:: ../_scripts/upload.sh
+   :language: bash
 
-Response::
+Content GET Response::
 
     {
-        "_href": "/pulp/api/v3/artifacts/1/",
-        ...
+        "_artifact": null,
+        "_created": "2019-07-25T13:57:55.178993Z",
+        "_href": "/pulp/api/v3/content/python/packages/6172ff0f-3e11-4b5f-8460-bd6a72616747/",
+        "_type": "python.python",
+        "author": "",
+        "author_email": "",
+        "classifiers": [],
+        "description": "",
+        "download_url": "",
+        "filename": "shelf_reader-0.1-py2-none-any.whl",
+        "home_page": "",
+        "keywords": "",
+        "license": "",
+        "maintainer": "",
+        "maintainer_email": "",
+        "metadata_version": "",
+        "name": "[]",
+        "obsoletes_dist": "[]",
+        "packagetype": "bdist_wheel",
+        "platform": "",
+        "project_url": "",
+        "provides_dist": "[]",
+        "requires_dist": "[]",
+        "requires_external": "[]",
+        "requires_python": "",
+        "summary": "",
+        "supported_platform": "",
+        "version": "0.1"
     }
 
+Reference: `Python Content Usage <../restapi.html#tag/content>`_
 
-Reference (pulpcore): `Artifact API Usage
-<https://docs.pulpproject.org/en/3.0/nightly/restapi.html#tag/artifacts>`_
-
-Create content from an artifact
--------------------------------
+Add content to a repository during one-shot upload
+--------------------------------------------------
 
-Now that Pulp has the wheel, its time to make it into a unit of content. The python plugin will
-inspect the file and populate its metadata::
+One-shot upload can also optionally add the content being created to a repository you specify.
 
-    $ http POST $BASE_ADDR/pulp/api/v3/content/python/packages/ _artifact=$ARTIFACT_HREF filename=shelf_reader-0.1-py2-none-any.whl
+.. literalinclude:: ../_scripts/upload_with_repo.sh
+   :language: bash
 
-Response::
+Repository GET Response::
 
     {
-        "_href": "/pulp/api/v3/content/python/packages/1/",
-        "_artifact": "/pulp/api/v3/artifacts/1/",
-        "digest": "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c",
-        "filename": "shelf_reader-0.1-py2-none-any.whl",
-        "type": "python"
+        "_created": "2019-07-25T14:03:48.378437Z",
+        "_href": "/pulp/api/v3/repositories/135f468f-0c61-4337-9f37-0cd911244bec/versions/1/",
+        "base_version": null,
+        "content_summary": {
+            "added": {
+                "python.python": {
+                    "count": 1,
+                    "href": "/pulp/api/v3/content/python/packages/?repository_version_added=/pulp/api/v3/repositories/135f468f-0c61-4337-9f37-0cd911244bec/versions/1/"
+                }
+            },
+            "present": {
+                "python.python": {
+                    "count": 1,
+                    "href": "/pulp/api/v3/content/python/packages/?repository_version=/pulp/api/v3/repositories/135f468f-0c61-4337-9f37-0cd911244bec/versions/1/"
+                }
+            },
+            "removed": {}
+        },
+        "number": 1
     }
 
-Create a variable for convenience::
-
-    $ export CONTENT_HREF=$(http $BASE_ADDR/pulp/api/v3/content/python/packages/ | jq -r '.results[] | select(.filename == "shelf_reader-0.1-py2-none-any.whl") | ._href')
-
-Reference: `Python Content API Usage <../restapi.html#tag/content>`_
 
-Add content to a repository
----------------------------
+Reference: `Python Repository Usage <../restapi.html#tag/repositories>`_
 
-See :ref:`add-remove`
+For other ways to add content to a repository, see :ref:`add-remove`

pulp_python/app/serializers.py

@@ -5,6 +5,7 @@
 from rest_framework import serializers
 
 from pulpcore.plugin import models as core_models
+from pulpcore.plugin.models import Repository
 from pulpcore.plugin import serializers as core_serializers
 
 from pulp_python.app import models as python_models
@@ -214,6 +215,23 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class PythonOneShotUploadSerializer(serializers.Serializer):
+    """
+    A Serializer for PythonOneShotUpload.
+    """
+
+    repository = serializers.HyperlinkedRelatedField(
+        help_text=_('A URI of the repository.'),
+        required=False,
+        queryset=Repository.objects.all(),
+        view_name='repositories-detail',
+    )
+    file = serializers.FileField(
+        help_text=_("The python file (i.e. .whl or .tar.gz)."),
+        required=True,
+    )
+
+
 class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer):
     """
     A Serializer for PythonPackageContent.

pulp_python/app/tasks/upload.py

@@ -0,0 +1,80 @@
+import os
+from gettext import gettext as _
+import pkginfo
+import shutil
+import tempfile
+
+from pulpcore.plugin.models import Artifact, CreatedResource, Repository, RepositoryVersion
+from rest_framework import serializers
+
+from pulp_python.app.models import PythonPackageContent
+from pulp_python.app.utils import parse_project_metadata
+
+
+DIST_EXTENSIONS = {
+    ".whl": "bdist_wheel",
+    ".exe": "bdist_wininst",
+    ".egg": "bdist_egg",
+    ".tar.bz2": "sdist",
+    ".tar.gz": "sdist",
+    ".zip": "sdist",
+}
+
+DIST_TYPES = {
+    "bdist_wheel": pkginfo.Wheel,
+    "bdist_wininst": pkginfo.Distribution,
+    "bdist_egg": pkginfo.BDist,
+    "sdist": pkginfo.SDist,
+}
+
+
+def one_shot_upload(artifact_pk, filename, repository_pk=None):
+    """
+    One shot upload for pulp_python
+
+    Args:
+        artifact_pk: validated artifact
+        filename: file name
+        repository_pk: optional repository to add Content to
+    """
+    # iterate through extensions since splitext does not support things like .tar.gz
+    for ext, packagetype in DIST_EXTENSIONS.items():
+        if filename.endswith(ext):
+            # Copy file to a temp directory under the user provided filename, we do this
+            # because pkginfo validates that the filename has a valid extension before
+            # reading it
+            with tempfile.TemporaryDirectory() as td:
+                temp_path = os.path.join(td, filename)
+                artifact = Artifact.objects.get(pk=artifact_pk)
+                shutil.copy2(artifact.file.path, temp_path)
+                metadata = DIST_TYPES[packagetype](temp_path)
+                metadata.packagetype = packagetype
+                break
+    else:
+        raise serializers.ValidationError(_(
+            "Extension on {} is not a valid python extension "
+            "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename)
+        )
+    data = parse_project_metadata(vars(metadata))
+    data['classifiers'] = [{'name': classifier} for classifier in metadata.classifiers]
+    data['packagetype'] = metadata.packagetype
+    data['version'] = metadata.version
+    data['filename'] = filename
+    data['_relative_path'] = filename
+
+    new_content = PythonPackageContent.objects.create(
+        filename=filename,
+        packagetype=metadata.packagetype,
+        name=data['classifiers'],
+        version=data['version']
+    )
+
+    queryset = PythonPackageContent.objects.filter(pk=new_content.pk)
+
+    if repository_pk:
+        repository = Repository.objects.get(pk=repository_pk)
+        with RepositoryVersion.create(repository) as new_version:
+            new_version.add_content(queryset)
+
+    resource = CreatedResource(content_object=new_content)
+    resource.save()

pulp_python/app/urls.py

@@ -0,0 +1,8 @@
+from django.conf.urls import url
+
+from .viewsets import PythonOneShotUploadViewSet
+
+
+urlpatterns = [
+    url(r'python/upload/$', PythonOneShotUploadViewSet.as_view({'post': 'create'}))
+]