submit endpoint (#4)

* update wort cli, expose new submit endpoint in API
* compressed sigs in S3
* fix shell
* status in response

Cargo.toml

@@ -10,4 +10,7 @@ license = "BSD-3-Clause"
 
 [dependencies]
 clap = { version = "~2.32", features = ["yaml"] }
-reqwest = "0.8"
+reqwest = "^0.9"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0.2"

src/main.rs

@@ -1,26 +1,64 @@
 #[macro_use]
 extern crate clap;
 extern crate reqwest;
+#[macro_use]
+extern crate serde_derive;
 
 use std::error::Error;
 
 use clap::App;
 
-fn viewer(db: &str, dataset_id: &str) -> Result<(), Box<Error>> {
-    let url = format!("https://wort.oxli.org/view/{}/{}", db, dataset_id);
+const BASEURL: &'static str = "https://wort.oxli.org/v1";
+
+#[derive(Debug, Deserialize)]
+struct Response {
+    status: String,
+}
+
+fn view(db: &str, dataset_id: &str) -> Result<(), Box<Error>> {
+    let url = format!("{}/view/{}/{}", BASEURL, db, dataset_id);
     let mut res = reqwest::get(&url)?;
     std::io::copy(&mut res, &mut std::io::stdout())?;
     Ok(())
 }
 
-fn main() {
+fn submit(db: String, dataset_id: String, token: &str, filename: &str) -> Result<(), Box<Error>> {
+    let form = reqwest::multipart::Form::new().file("file", filename)?;
+
+    let url = format!("{}/submit/{}/{}", BASEURL, db, dataset_id);
+
+    let client = reqwest::Client::new();
+    let mut res = client
+        .post(&url)
+        .bearer_auth(token)
+        .multipart(form)
+        .send()?;
+    println!("{}", res.json::<Response>()?.status);
+    Ok(())
+}
+
+fn main() -> Result<(), Box<Error>> {
     let yml = load_yaml!("wort.yml");
     let m = App::from_yaml(yml).get_matches();
 
-    if let Some(cmd) = m.subcommand_matches("viewer") {
-        viewer(
-            cmd.value_of("database").unwrap(),
-            cmd.value_of("dataset_id").unwrap(),
-        );
+    match m.subcommand_name() {
+        Some("view") => {
+            let cmd = m.subcommand_matches("view").unwrap();
+            view(
+                cmd.value_of("database").unwrap(),
+                cmd.value_of("dataset_id").unwrap(),
+            )
+        }
+        Some("submit") => {
+            let cmd = m.subcommand_matches("submit").unwrap();
+            submit(
+                cmd.value_of("database").unwrap().into(),
+                cmd.value_of("dataset_id").unwrap().into(),
+                cmd.value_of("token").unwrap(),
+                cmd.value_of("signature").unwrap(),
+            )
+        }
+        None => Ok(()), // TODO: should be error
+        _ => Ok(()),    // TODO: should be error
     }
 }

src/wort.yml

@@ -1,13 +1,13 @@
 name: wort
-version: "0.1.0"
+version: "0.2.0"
 about: Interact with the wort API from the command line
 author: Luiz Irber <wort@luizirber.org>
 
 settings:
     - ArgRequiredElseHelp
 
 subcommands:
-    - viewer:
+    - view:
         about: view a signature
         settings:
             - ArgRequiredElseHelp
@@ -23,3 +23,30 @@ subcommands:
             - dataset_id:
                 help: ID of the dataset in the DB
                 index: 1
+    - submit:
+        about: submit a signature
+        settings:
+            - ArgRequiredElseHelp
+        args:
+            - token:
+                short: t
+                help: user name
+                required: true
+                takes_value: true
+            - database:
+                short: d
+                help: database id
+                takes_value: true
+                default_value: sra
+                possible_values:
+                  - img
+                  - sra
+            - dataset_id:
+                short: i
+                help: ID of the dataset in the DB
+                required: true
+                takes_value: true
+            - signature:
+                help: sourmash signature file
+                index: 1
+                required: true

wort/api.yaml

@@ -58,6 +58,16 @@ paths:
       responses:
         200:
           description: token revoked
+  '/submit/{public_db}/{dataset_id}':
+    post:
+      summary: Submit a signature
+      operationId: wort.blueprints.submit.views.submit_sigs
+      parameters:
+        - $ref: '#/parameters/public_db'
+        - $ref: '#/parameters/dataset_id'
+      responses:
+        202:
+          description: Signature accepted
 
 parameters:
   sra_id:

wort/blueprints/api/tokens.py

@@ -5,17 +5,15 @@
 from wort.ext import basic_auth, token_auth, db
 
 
-# @api.route("/tokens", methods=["POST"])
 @basic_auth.login_required
 def get_token():
     token = g.current_user.get_token(expires_in=86400)
     db.session.commit()
-    return jsonify({"token": token})
+    return jsonify({"status": "OK", "token": token})
 
 
-# @api.route("/tokens", methods=["DELETE"])
 @token_auth.login_required
 def revoke_token():
     g.current_user.get_token()
     db.session.commit()
-    return "", 204
+    return jsonify({"status": "OK"}), 204

wort/blueprints/compute/tasks.py

@@ -1,6 +1,9 @@
+import gzip
+from io import BytesIO
 import os
 from subprocess import CalledProcessError
-from tempfile import NamedTemporaryFile, TemporaryDirectory
+import shutil
+from tempfile import NamedTemporaryFile
 
 from celery.exceptions import Ignore
 
@@ -15,6 +18,7 @@ def compute(sra_id):
     import botocore
     from snakemake import shell
 
+    conn = boto3.client("s3")
     s3 = boto3.resource("s3")
 
     key_path = os.path.join("sigs", sra_id + ".sig")
@@ -50,12 +54,19 @@ def compute(sra_id):
             if e.returncode != 141:
                 raise e
 
-        # save to S3
-        key = s3.Object("wort-sra", key_path)
         f.seek(0)
-        # TODO: compress using gzip here!
-        # https://gist.github.com/veselosky/9427faa38cee75cd8e27
-        key.upload_fileobj(f)
+
+        compressed_fp = BytesIO()
+        with gzip.GzipFile(fileobj=compressed_fp, mode="wb") as gz:
+            shutil.copyfileobj(f, gz)
+
+        conn.put_object(
+            Body=compressed_fp.getvalue(),
+            Bucket="wort-sra",
+            Key=key_path,
+            ContentType="application/json",
+            ContentEncoding="gzip",
+        )
 
 
 @celery.task

wort/blueprints/submit/views.py

@@ -1,26 +1,39 @@
-from flask import Blueprint, request, jsonify, flash
+import gzip
+from io import BytesIO
+import shutil
+
+from flask import Blueprint, request, jsonify, g
+from wort.blueprints.api.auth import token_auth
+
 
 submit = Blueprint("submit", __name__, template_folder="templates")
 
 
-@submit.route("/submit", methods=["GET", "POST"])
-def submit_sigs():
-    if request.method == "POST":
-        if "file" not in request.files:
-            flash("No file part")
-            return redirect(request.url)
-
-        f = request.files["file"]
-        print(f, request.form["public_url"])
-        return jsonify({}), 200
-
-    return """
-    <!doctype html>
-    <title>Upload new File</title>
-    <h1>Upload new File</h1>
-    <form method=post enctype=multipart/form-data>
-      <p><input type=file name=file>
-         <input type=text name=public_url>
-         <input type=submit value=Upload>
-    </form>
-    """
+@token_auth.login_required
+def submit_sigs(public_db, dataset_id):
+
+    if public_db not in ("sra", "img"):
+        return "Database not supported", 404
+
+    import boto3
+
+    conn = boto3.client("s3")
+
+    username = g.current_user.username
+    key = f"{username}/{dataset_id}.sig"
+
+    file = request.files["file"]
+    compressed_fp = BytesIO()
+    # TODO: if it's already gzipped, don't compress it
+    with gzip.GzipFile(fileobj=compressed_fp, mode="wb") as gz:
+        shutil.copyfileobj(file.stream, gz)
+
+    conn.put_object(
+        Body=compressed_fp.getvalue(),
+        Bucket=f"wort-submitted-{public_db}",
+        Key=key,
+        ContentType="application/json",
+        ContentEncoding="gzip",
+    )
+
+    return jsonify({"status": "Signature accepted"}), 202

wort/blueprints/viewer/views.py

@@ -14,20 +14,14 @@ def view_s3(public_db, dataset_id):
     conn = boto3.client("s3")
 
     key = f"sigs/{dataset_id}.sig"
-    # TODO: we don't really need this, I just copied the signatures improperly
-    # to the bucket...
-    if public_db == "img":
-        key = f"{dataset_id}.sig"
-
-    url = conn.generate_presigned_url(
-        "get_object",
-        Params={
-            "Bucket": f"wort-{public_db}",
-            "Key": key,
-            "ResponseContentType": "application/json",
-            # 'ResponseContentEncoding': 'gzip',
-        },
-        ExpiresIn=100,
-    )
+
+    params = {
+        "Bucket": f"wort-{public_db}",
+        "Key": key,
+        "ResponseContentType": "application/json",
+        "ResponseContentEncoding": "gzip",
+    }
+
+    url = conn.generate_presigned_url("get_object", Params=params, ExpiresIn=100)
 
     return redirect(url)