OS Data Refactor - Updated API Workflow Spec

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.30.5] - 2022-03-11
+
+### Updated
+
+- Updated Workflow endpoints & API spec to support upload & download of result objects as pickle files
+
 ## [0.30.4] - 2022-03-11
 
 ### Fixed

VERSION

@@ -1 +1 @@
-0.30.4
+0.30.5

refactor/data/app/api/api_v0/endpoints/workflow.py

@@ -18,7 +18,7 @@
 #
 # Relief from the License may be granted by purchasing a commercial license.
 
-
+import io
 from typing import Any, Union
 
 from app.schemas.common import HTTPExceptionSchema
@@ -29,78 +29,73 @@
     ResultPickle,
     UpdateResultResponse,
 )
-from fastapi import APIRouter
+from fastapi import APIRouter, UploadFile, HTTPException
+from fastapi.responses import FileResponse, StreamingResponse
 
 router = APIRouter()
 
-# For now we don't return this, instead just a bytes-like pickle object
-mock_result_object_instance = {
-    "dispatch_id": "",
-    "results_dir": "",
-    "status": "COMPLETED",
-    "graph": {
-        "links": [{"edge_name": "data", "param_type": "kwarg", "source": 0, "target": 1}],
-        "nodes": [
-            {
-                "name": "load_data",
-                "metadata": {"executor": "<LocalExecutor>"},
-                "function_string": "@ct.electron\ndef load_data():\n    iris = datasets.load_iris()\n    perm = permutation(iris.target.size)\n    iris.data = iris.data[perm]\n    iris.target = iris.target[perm]\n    return iris.data, iris.target\n\n\n",
-                "start_time": "2022-02-28T23:21:17.844268+00:00",
-                "end_time": "2022-02-28T23:21:17.853741+00:00",
-                "status": "COMPLETED",
-                "output": ["[[5.  3.4 1.5 0.2]]"],
-                "error": None,
-                "sublattice_result": None,
-                "stdout": "",
-                "stderr": "",
-                "id": 0,
-                "doc": None,
-                "kwargs": None,
-            }
-        ],
-    },
-}
-
 
 @router.get(
     "/results/{dispatch_id}",
     status_code=200,
-    response_model=ResultPickle,
+    response_class=FileResponse,
     responses={
-        404: {"model": HTTPExceptionSchema, "description": "The result was not found"},
-    },
+        404: {"model": HTTPExceptionSchema, "description": "Result was not found"},
+        200: {
+            "content": {"application/octet-stream": {}},
+            "description": "Return binary content of file.",
+        }
+    }
 )
 def get_result(
     *,
     dispatch_id: str,
 ) -> Any:
     """
-    Get a result object
+    Get a result object as pickle file
     """
+    result: bytes = b'\x00\xF0'
+    # update logic to db lookup
+    if not dispatch_id:
+        raise HTTPException(status_code=404, detail="Result not found")
+    return StreamingResponse(io.BytesIO(result), media_type="application/octet-stream")
 
-    return {"result_object": b"result pickle object"}
 
 
 @router.post("/results", status_code=200, response_model=InsertResultResponse)
 def insert_result(
     *,
-    result_object: Union[Result, ResultPickle],
+    result_pkl_file: UploadFile,
 ) -> Any:
     """
-    Insert a result object
+    Submit pickled result file
     """
-
-    return {"response": "Result successfully added to db"}
+    return {
+        "dispatch_id": "e4efd26c-240d-4ab1-9826-26ada91e429f"
+    }
 
 
-@router.put("/results/{dispatch_id}", status_code=200, response_model=UpdateResultResponse)
+@router.put(
+    "/results/{dispatch_id}", 
+    status_code=200,
+    responses={
+        404: {"model": HTTPExceptionSchema, "description": "Result was not found"},
+        200: {
+            "model": UpdateResultResponse,
+            "description": "Return message indicating success of updating task",
+        }
+})
 def update_result(
     *,
     dispatch_id: str,
-    task: Node,
+    task: Node
 ) -> Any:
     """
-    Update a result object
+    Update a result object's task
     """
-
-    return {"response": "Task updated successfully"}
+    # update logic to db lookup
+    if not dispatch_id:
+        raise HTTPException(status_code=404, detail="Result not found")
+    return {
+        "response": "Task updated successfully"
+    }

refactor/data/app/schemas/workflow.py

@@ -60,7 +60,7 @@ class Result(BaseModel):
 
 
 class InsertResultResponse(BaseModel):
-    response: str
+    dispatch_id: str
 
 
 class UpdateResultResponse(BaseModel):

refactor/data/openapi.yaml

@@ -8,7 +8,7 @@ paths:
       tags:
         - Workflow
       summary: Get Result
-      description: Get a result object
+      description: Get a result object from the database
       operationId: get_result_api_v0_workflow_results__dispatch_id__get
       parameters:
         - required: true
@@ -19,13 +19,11 @@ paths:
           in: path
       responses:
         '200':
-          description: Successful Response
+          description: Return binary content of file.
           content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ResultPickle'
+            application/octet-stream: {}
         '404':
-          description: The result was not found
+          description: Result was not found
           content:
             application/json:
               schema:
@@ -40,7 +38,7 @@ paths:
       tags:
         - Workflow
       summary: Update Result
-      description: Update a result object
+      description: Update a result object's task
       operationId: update_result_api_v0_workflow_results__dispatch_id__put
       parameters:
         - required: true
@@ -57,11 +55,17 @@ paths:
         required: true
       responses:
         '200':
-          description: Successful Response
+          description: Return message indicating success of updating task
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/UpdateResultResponse'
+        '404':
+          description: Result was not found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPExceptionSchema'
         '422':
           description: Validation Error
           content:
@@ -73,16 +77,13 @@ paths:
       tags:
         - Workflow
       summary: Insert Result
-      description: Insert a result object
+      description: Insert a result object to database by uploading pickled result object
       operationId: insert_result_api_v0_workflow_results_post
       requestBody:
         content:
-          application/json:
+          multipart/form-data:
             schema:
-              title: Result Object
-              anyOf:
-                - $ref: '#/components/schemas/Result'
-                - $ref: '#/components/schemas/ResultPickle'
+              $ref: '#/components/schemas/Body_insert_result_api_v0_workflow_results_post'
         required: true
       responses:
         '200':
@@ -156,6 +157,16 @@ paths:
                 $ref: '#/components/schemas/HTTPValidationError'
 components:
   schemas:
+    Body_insert_result_api_v0_workflow_results_post:
+      title: Body_insert_result_api_v0_workflow_results_post
+      required:
+        - result_pkl_file
+      type: object
+      properties:
+        result_pkl_file:
+          title: Result Pkl File
+          type: string
+          format: binary
     Body_upload_file_api_v0_fs_upload_post:
       title: Body_upload_file_api_v0_fs_upload_post
       required:
@@ -166,23 +177,6 @@ components:
           title: File
           type: string
           format: binary
-    Graph:
-      title: Graph
-      required:
-        - nodes
-        - links
-      type: object
-      properties:
-        nodes:
-          title: Nodes
-          type: array
-          items:
-            $ref: '#/components/schemas/Node'
-        links:
-          title: Links
-          type: array
-          items:
-            $ref: '#/components/schemas/Link'
     HTTPExceptionSchema:
       title: HTTPExceptionSchema
       required:
@@ -204,33 +198,12 @@ components:
     InsertResultResponse:
       title: InsertResultResponse
       required:
-        - response
-      type: object
-      properties:
-        response:
-          title: Response
-          type: string
-    Link:
-      title: Link
-      required:
-        - edge_name
-        - param_type
-        - source
-        - target
+        - dispatch_id
       type: object
       properties:
-        edge_name:
-          title: Edge Name
-          type: string
-        param_type:
-          title: Param Type
+        dispatch_id:
+          title: Dispatch Id
           type: string
-        source:
-          title: Source
-          type: integer
-        target:
-          title: Target
-          type: integer
     Node:
       title: Node
       required:
@@ -271,36 +244,6 @@ components:
         id:
           title: Id
           type: integer
-    Result:
-      title: Result
-      required:
-        - dispatch_id
-        - results_dir
-        - status
-        - graph
-      type: object
-      properties:
-        dispatch_id:
-          title: Dispatch Id
-          type: string
-        results_dir:
-          title: Results Dir
-          type: string
-        status:
-          title: Status
-          type: string
-        graph:
-          $ref: '#/components/schemas/Graph'
-    ResultPickle:
-      title: ResultPickle
-      required:
-        - result_object
-      type: object
-      properties:
-        result_object:
-          title: Result Object
-          type: string
-          format: binary
     UpdateResultResponse:
       title: UpdateResultResponse
       required: