From 6ad26dd932ec17c1249f46577e2aa332e7676fd8 Mon Sep 17 00:00:00 2001 From: Luiz Irber Date: Fri, 5 Oct 2018 17:59:32 +0000 Subject: [PATCH] submit endpoint (#4) * update wort cli, expose new submit endpoint in API * compressed sigs in S3 * fix shell * status in response --- Cargo.toml | 5 ++- src/main.rs | 54 +++++++++++++++++++++++++----- src/wort.yml | 31 +++++++++++++++-- wort/api.yaml | 10 ++++++ wort/blueprints/api/tokens.py | 6 ++-- wort/blueprints/compute/tasks.py | 23 +++++++++---- wort/blueprints/submit/views.py | 57 ++++++++++++++++++++------------ wort/blueprints/viewer/views.py | 24 +++++--------- 8 files changed, 152 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e6509a3..7a34865 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,7 @@ license = "BSD-3-Clause" [dependencies] clap = { version = "~2.32", features = ["yaml"] } -reqwest = "0.8" +reqwest = "^0.9" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0.2" diff --git a/src/main.rs b/src/main.rs index ea0c453..8c710dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,26 +1,64 @@ #[macro_use] extern crate clap; extern crate reqwest; +#[macro_use] +extern crate serde_derive; use std::error::Error; use clap::App; -fn viewer(db: &str, dataset_id: &str) -> Result<(), Box> { - let url = format!("https://wort.oxli.org/view/{}/{}", db, dataset_id); +const BASEURL: &'static str = "https://wort.oxli.org/v1"; + +#[derive(Debug, Deserialize)] +struct Response { + status: String, +} + +fn view(db: &str, dataset_id: &str) -> Result<(), Box> { + let url = format!("{}/view/{}/{}", BASEURL, db, dataset_id); let mut res = reqwest::get(&url)?; std::io::copy(&mut res, &mut std::io::stdout())?; Ok(()) } -fn main() { +fn submit(db: String, dataset_id: String, token: &str, filename: &str) -> Result<(), Box> { + let form = reqwest::multipart::Form::new().file("file", filename)?; + + let url = format!("{}/submit/{}/{}", BASEURL, db, dataset_id); + + let client = reqwest::Client::new(); + let mut res = client + .post(&url) + .bearer_auth(token) + .multipart(form) + .send()?; + println!("{}", res.json::()?.status); + Ok(()) +} + +fn main() -> Result<(), Box> { let yml = load_yaml!("wort.yml"); let m = App::from_yaml(yml).get_matches(); - if let Some(cmd) = m.subcommand_matches("viewer") { - viewer( - cmd.value_of("database").unwrap(), - cmd.value_of("dataset_id").unwrap(), - ); + match m.subcommand_name() { + Some("view") => { + let cmd = m.subcommand_matches("view").unwrap(); + view( + cmd.value_of("database").unwrap(), + cmd.value_of("dataset_id").unwrap(), + ) + } + Some("submit") => { + let cmd = m.subcommand_matches("submit").unwrap(); + submit( + cmd.value_of("database").unwrap().into(), + cmd.value_of("dataset_id").unwrap().into(), + cmd.value_of("token").unwrap(), + cmd.value_of("signature").unwrap(), + ) + } + None => Ok(()), // TODO: should be error + _ => Ok(()), // TODO: should be error } } diff --git a/src/wort.yml b/src/wort.yml index e4cd26d..eef22e7 100644 --- a/src/wort.yml +++ b/src/wort.yml @@ -1,5 +1,5 @@ name: wort -version: "0.1.0" +version: "0.2.0" about: Interact with the wort API from the command line author: Luiz Irber @@ -7,7 +7,7 @@ settings: - ArgRequiredElseHelp subcommands: - - viewer: + - view: about: view a signature settings: - ArgRequiredElseHelp @@ -23,3 +23,30 @@ subcommands: - dataset_id: help: ID of the dataset in the DB index: 1 + - submit: + about: submit a signature + settings: + - ArgRequiredElseHelp + args: + - token: + short: t + help: user name + required: true + takes_value: true + - database: + short: d + help: database id + takes_value: true + default_value: sra + possible_values: + - img + - sra + - dataset_id: + short: i + help: ID of the dataset in the DB + required: true + takes_value: true + - signature: + help: sourmash signature file + index: 1 + required: true diff --git a/wort/api.yaml b/wort/api.yaml index 1c1ba9a..379e488 100644 --- a/wort/api.yaml +++ b/wort/api.yaml @@ -58,6 +58,16 @@ paths: responses: 200: description: token revoked + '/submit/{public_db}/{dataset_id}': + post: + summary: Submit a signature + operationId: wort.blueprints.submit.views.submit_sigs + parameters: + - $ref: '#/parameters/public_db' + - $ref: '#/parameters/dataset_id' + responses: + 202: + description: Signature accepted parameters: sra_id: diff --git a/wort/blueprints/api/tokens.py b/wort/blueprints/api/tokens.py index face0b7..00af9dd 100644 --- a/wort/blueprints/api/tokens.py +++ b/wort/blueprints/api/tokens.py @@ -5,17 +5,15 @@ from wort.ext import basic_auth, token_auth, db -# @api.route("/tokens", methods=["POST"]) @basic_auth.login_required def get_token(): token = g.current_user.get_token(expires_in=86400) db.session.commit() - return jsonify({"token": token}) + return jsonify({"status": "OK", "token": token}) -# @api.route("/tokens", methods=["DELETE"]) @token_auth.login_required def revoke_token(): g.current_user.get_token() db.session.commit() - return "", 204 + return jsonify({"status": "OK"}), 204 diff --git a/wort/blueprints/compute/tasks.py b/wort/blueprints/compute/tasks.py index bcb6b45..fda3e3b 100644 --- a/wort/blueprints/compute/tasks.py +++ b/wort/blueprints/compute/tasks.py @@ -1,6 +1,9 @@ +import gzip +from io import BytesIO import os from subprocess import CalledProcessError -from tempfile import NamedTemporaryFile, TemporaryDirectory +import shutil +from tempfile import NamedTemporaryFile from celery.exceptions import Ignore @@ -15,6 +18,7 @@ def compute(sra_id): import botocore from snakemake import shell + conn = boto3.client("s3") s3 = boto3.resource("s3") key_path = os.path.join("sigs", sra_id + ".sig") @@ -50,12 +54,19 @@ def compute(sra_id): if e.returncode != 141: raise e - # save to S3 - key = s3.Object("wort-sra", key_path) f.seek(0) - # TODO: compress using gzip here! - # https://gist.github.com/veselosky/9427faa38cee75cd8e27 - key.upload_fileobj(f) + + compressed_fp = BytesIO() + with gzip.GzipFile(fileobj=compressed_fp, mode="wb") as gz: + shutil.copyfileobj(f, gz) + + conn.put_object( + Body=compressed_fp.getvalue(), + Bucket="wort-sra", + Key=key_path, + ContentType="application/json", + ContentEncoding="gzip", + ) @celery.task diff --git a/wort/blueprints/submit/views.py b/wort/blueprints/submit/views.py index a3bd8cb..d65bcd4 100644 --- a/wort/blueprints/submit/views.py +++ b/wort/blueprints/submit/views.py @@ -1,26 +1,39 @@ -from flask import Blueprint, request, jsonify, flash +import gzip +from io import BytesIO +import shutil + +from flask import Blueprint, request, jsonify, g +from wort.blueprints.api.auth import token_auth + submit = Blueprint("submit", __name__, template_folder="templates") -@submit.route("/submit", methods=["GET", "POST"]) -def submit_sigs(): - if request.method == "POST": - if "file" not in request.files: - flash("No file part") - return redirect(request.url) - - f = request.files["file"] - print(f, request.form["public_url"]) - return jsonify({}), 200 - - return """ - - Upload new File -

Upload new File

-
-

- - -

- """ +@token_auth.login_required +def submit_sigs(public_db, dataset_id): + + if public_db not in ("sra", "img"): + return "Database not supported", 404 + + import boto3 + + conn = boto3.client("s3") + + username = g.current_user.username + key = f"{username}/{dataset_id}.sig" + + file = request.files["file"] + compressed_fp = BytesIO() + # TODO: if it's already gzipped, don't compress it + with gzip.GzipFile(fileobj=compressed_fp, mode="wb") as gz: + shutil.copyfileobj(file.stream, gz) + + conn.put_object( + Body=compressed_fp.getvalue(), + Bucket=f"wort-submitted-{public_db}", + Key=key, + ContentType="application/json", + ContentEncoding="gzip", + ) + + return jsonify({"status": "Signature accepted"}), 202 diff --git a/wort/blueprints/viewer/views.py b/wort/blueprints/viewer/views.py index dd2d969..671bf65 100644 --- a/wort/blueprints/viewer/views.py +++ b/wort/blueprints/viewer/views.py @@ -14,20 +14,14 @@ def view_s3(public_db, dataset_id): conn = boto3.client("s3") key = f"sigs/{dataset_id}.sig" - # TODO: we don't really need this, I just copied the signatures improperly - # to the bucket... - if public_db == "img": - key = f"{dataset_id}.sig" - - url = conn.generate_presigned_url( - "get_object", - Params={ - "Bucket": f"wort-{public_db}", - "Key": key, - "ResponseContentType": "application/json", - # 'ResponseContentEncoding': 'gzip', - }, - ExpiresIn=100, - ) + + params = { + "Bucket": f"wort-{public_db}", + "Key": key, + "ResponseContentType": "application/json", + "ResponseContentEncoding": "gzip", + } + + url = conn.generate_presigned_url("get_object", Params=params, ExpiresIn=100) return redirect(url)