API Endpoints (#384)

* Initial script api work

* Initial script api

* Refactor form update logic

* Add status codes to api

* Add job submission and detail endpoints

* Remove __all__ from access message in json api

* Add api based script addition

* ADd check for MultiValueDict

* Fix MultiValueDict import

* Fix validate form

* USe ensure_list

* Refactor, more tests

* Test api

* rename tst

* test stubs

* More tests

* More tests, docs

* Move api key location

* Fix anon user

* more doc updates

* Add example using files

* Fix request parsing

* remove decoding

docker/celeryconfig.py

@@ -1,4 +1,4 @@
-broker_url = "amqp://guest@rabbit"
+broker_url = "amqp://guest@rabbit:5672"
 track_started = True
 send_events = True
 imports = ("wooey.tasks",)

docker/docker-compose.yml

@@ -23,7 +23,7 @@ services:
     depends_on:
       - rabbit
       - db
-    command: watchmedo auto-restart --directory=$BUILD_DIR/wooey --recursive --ignore-patterns="*.pyc" -- celery worker -A $WOOEY_PROJECT -c 4 -B -l debug -s schedule
+    command: watchmedo auto-restart --directory=$BUILD_DIR/wooey --recursive --ignore-patterns="*.pyc" -- celery -A $WOOEY_PROJECT worker -c 4 -B -l debug -s schedule
 
   rabbit:
     image: rabbitmq:3.9.29-management-alpine

docs/api.rst

@@ -0,0 +1,128 @@
+Wooey API
+=========
+
+Wooey's API allows for programmatic access to manage scripts as well as submit and query jobs.
+All use of the API requires a user to be authenticated via :ref:`API keys <api_keys>` and
+the API to be enabled by setting `WOOEY_ENABLE_API_KEYS` in your user_settings.py file.
+
+Script Management API
+~~~~~~~~~~~~~~~~~~~~~
+
+Adding and updating a script use the same endpoint, **api/scripts/v1/add-or-update/**.
+
+.. code-block:: python
+
+    import requests
+
+    response = requests.post(
+      'https://wooey.fly.dev/api/scripts/v1/add-or-update/',
+      data={
+        "group": "The script group", # optional
+        "script-name": open("path_to_script.py", "rb")
+      },
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+
+For updating an existing script, the same code can be used with the new version. By default,
+updating a script will make that version the default version to run for submissions. To disable
+this, add `default: False` to the payload. Multiple scripts can be uploaded at once by simply
+providing multiple files. The name of the script is the key used for the file. Thus, for this example
+our script name would be `script-name`.
+
+
+Job API
+~~~~~~~
+
+Creating a new Job
+##################
+
+A script can be ran via the **api/scripts/v1/<script_slug>/submit/** endpoint. A `script_slug` is the
+script name, with any invalid url characters removed. This will normally be the lowercase version of the
+script's name, but can be found by looking at the url of a given script.
+
+.. image:: img/script_slug_example.png
+
+.. code-block:: python
+
+    import requests
+
+    response = requests.post(
+      'https://wooey.fly.dev/api/scripts/v1/cat-fetcher/submit/',
+      data={
+        "job_name": "test job",
+        "command": "--count 5 --breed bengal"
+      },
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+
+    # A valid response will contain
+    data = response.json()
+    # {"job_id": 123, "valid": True}
+
+For jobs that require files, the uploaded file can be provided and referenced in the `command` parameter. For example:
+
+.. code-block:: python
+
+    import requests
+
+    response = requests.post(
+      'https://wooey.fly.dev/api/scripts/v1/protein-translation/submit/',
+      data={
+        "job_name": "test job",
+        "command": "--fasta protein_sequences"
+      },
+      files={
+        "protein_sequences": open('./proteins.fasta')
+      },
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+
+Currently, this is only supported if the parameter is marked as a filetype (such as `here <https://docs.python.org/3/library/argparse.html#filetype-objects>`_).
+
+Querying Jobs
+#############
+
+A job can be queried by its id. While the UI allows sharing and management of jobs via a shareable UUID, that
+currently does not exist for Wooey's API as there is no public access permitted. **Importantly, these requests
+are GET requests**
+
+There are 2 endpoints for querying: **api/jobs/v1/<job_id>/status/** and **api/jobs/v1/<job_id>/details/**.
+
+**api/jobs/v1/<job_id>/status/** will provide information if the job is complete and should be used for polling.
+Once the job is complete, **api/jobs/v1/<job_id>/details/** will provide rich details about the job, including
+all assets generated by it and URLs to programatically download assets.
+
+.. code-block:: python
+
+    import requests
+
+    requests.get(
+      'https://wooey.fly.dev/api/jobs/v1/123/status/',
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+    # {"status": "running", "is_complete": False}
+    ...
+    requests.get(
+      'https://wooey.fly.dev/api/jobs/v1/123/status/',
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+    # {"status": "completed", "is_complete": True}
+    requests.get(
+      'https://wooey.fly.dev/api/jobs/v1/123/details/',
+      headers={'Authorization': 'Bearer your_token_here'},
+    )
+    # {
+    #    "status": "completed",
+    #    "is_complete": True,
+    #    "job_name": "test job",
+    #    "job_description": "",
+    #    "assets": [{"name": "assert 1", "url": "https://...", ...}],
+    #    "stdout": "This job's output, errors and other information would appear here",
+    #    "stderr": "This job's error output, errors and other information would appear here",
+    #    "uuid": "The sharable UUID, this can be used to provide someone a permalink to the UI view of the Job"
+    # }
+
+.. toctree::
+   :maxdepth: 1
+
+   api_keys

docs/index.rst

@@ -37,6 +37,8 @@ Getting Started
    running_wooey
    scripts
    wooey_ui
+   api
+   api_keys
    customizations
    remote
    upgrade_help

docs/wooey_ui.rst

@@ -43,8 +43,3 @@ to parse this information), updates to the script version will result in a new
 version being created. If a command line library doesn't support versioning
 or the version has not been updated in a script, the Script Iteration counter
 will be incremented.
-
-.. toctree::
-   :maxdepth: 2
-
-   api_keys

wooey/api/__init__.py

@@ -0,0 +1,8 @@
+from .jobs import (  # noqa: F401
+    job_details,
+    job_status,
+)
+from .scripts import (  # noqa: F401
+    add_or_update_script,
+    submit_script,
+)

wooey/api/forms.py

@@ -0,0 +1,19 @@
+from django import forms
+
+
+class SubmitForm(forms.Form):
+    job_name = forms.CharField()
+    job_description = forms.CharField(required=False)
+    version = forms.CharField(required=False)
+    iteration = forms.IntegerField(required=False)
+    command = forms.CharField(required=False)
+
+
+class AddScriptForm(forms.Form):
+    group = forms.CharField(required=False)
+    default = forms.NullBooleanField(required=False)
+
+    def clean_default(self):
+        if self.cleaned_data["default"] is None:
+            return True
+        return self.cleaned_data["default"]

wooey/api/jobs.py

@@ -0,0 +1,78 @@
+from django.http import JsonResponse
+from django.utils.encoding import force_str
+from django.utils.translation import gettext_lazy as _
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+
+from .. import models
+from ..utils import requires_login
+
+
+@csrf_exempt
+@require_http_methods(["GET"])
+@requires_login
+def job_status(request, job_id):
+    job = models.WooeyJob.objects.get(id=job_id)
+    if job.can_user_view(request.user):
+        return JsonResponse(
+            {
+                "status": job.status,
+                "is_complete": job.status in models.WooeyJob.TERMINAL_STATES,
+            }
+        )
+    else:
+        return JsonResponse(
+            {
+                "valid": False,
+                "errors": {
+                    "__all__": [
+                        force_str(_("You are not permitted to access this job."))
+                    ]
+                },
+            },
+            status=403,
+        )
+
+
+@csrf_exempt
+@require_http_methods(["GET"])
+@requires_login
+def job_details(request, job_id):
+    job = models.WooeyJob.objects.get(id=job_id)
+    if job.can_user_view(request.user):
+        assets = []
+        is_terminal = job.status in models.WooeyJob.TERMINAL_STATES
+        if is_terminal:
+            for asset in job.userfile_set.all():
+                assets.append(
+                    {
+                        "name": asset.filename,
+                        "url": request.build_absolute_uri(
+                            asset.system_file.filepath.url
+                        ),
+                    }
+                )
+        return JsonResponse(
+            {
+                "status": job.status,
+                "is_complete": is_terminal,
+                "uuid": job.uuid,
+                "job_name": job.job_name,
+                "job_description": job.job_description,
+                "stdout": job.stdout,
+                "stderr": job.stderr,
+                "assets": assets,
+            }
+        )
+    else:
+        return JsonResponse(
+            {
+                "valid": False,
+                "errors": {
+                    "__all__": [
+                        force_str(_("You are not permitted to access this job."))
+                    ]
+                },
+            },
+            status=403,
+        )