Merge pull request #786 from hammerlab/issue-621-

Issue 621 Use Voluptuous for object marshaling + object documentation

cycledash/__init__.py

@@ -1,4 +1,5 @@
-from flask import Flask, jsonify, request
+from flask import Flask, jsonify, request, make_response, current_app
+import flask.json
 from flask_sqlalchemy import SQLAlchemy
 from flask.ext import restful, login, bcrypt
 import humanize
@@ -21,6 +22,23 @@ def _configure_extensions(app):
     global db, api, login_manager, bcrypt
     db = SQLAlchemy(app)
     api = restful.Api(app, prefix='/api', catch_all_404s=True)
+
+    def output_json(data, status_code, headers=None):
+        """A JSON serializing request maker."""
+        settings = {}
+        if current_app.debug:
+            settings = {'indent': 4, 'sort_keys': True}
+        dumped = flask.json.dumps(data, **settings) + '\n'
+        resp = flask.make_response(dumped, status_code)
+        resp.headers.extend(headers or {})
+        return resp
+
+    # We primarily do this so that the JSON serializer in flask-restful can
+    # handle datetime objects. There's a hook to do this in the upcoming
+    # release of flask-restful, but as of 0.3.3, it's not exposed to the user.
+    api.representations = {
+        'application/json': output_json
+    }
     bcrypt = bcrypt.Bcrypt(app)
     login_manager = login.LoginManager()
     login_manager.init_app(app)

cycledash/api/__init__.py

@@ -1,9 +1,13 @@
+from collections import OrderedDict
 from flask import request
 import flask.ext.restful
 from flask.ext.login import current_user
+import functools
+import voluptuous
 
 from cycledash import login_manager
 from cycledash.auth import check_login
+from cycledash.helpers import prepare_request_data, camelcase_dict
 
 
 class Resource(flask.ext.restful.Resource, object):
@@ -33,4 +37,69 @@ def dispatch_request(self, *args, **kwargs):
         return super(Resource, self).dispatch_request(*args, **kwargs)
 
 
+def marshal(data, schema, envelope=None):
+    """Takes raw data and a schema to output, and applies the schema to the
+    object(s).
+
+    Args:
+       data: The actual object(s) from which the fields are taken from. A dict,
+             list, or tuple.
+       schema: A voluptuous.Schema of whose keys will make up the final
+               serialized response output
+       envelope: optional key that will be used to envelop the serialized
+                 response
+    """
+    items = None
+    if isinstance(data, (list, tuple)):
+        items = [marshal(d, schema) for d in data]
+    elif isinstance(data, dict):
+        items = schema(data)
+    else:
+        raise ValueError('`data` must be a list, tuple, or dict.')
+    if envelope:
+        items = [(envelope, items)]
+    return OrderedDict(items)
+
+
+def marshal_with(schema, envelope=None):
+    """Wraps flask-restful's marshal_with to transform the returned object to
+    have camelCased keys."""
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            resp = f(*args, **kwargs)
+            if not isinstance(resp, tuple):
+                content = resp
+                resp = (content, 200, {})
+            content = marshal(resp[0], schema, envelope=envelope)
+            content = camelcase_dict(content)
+            return (content,) + resp[1:]
+        return wrapper
+    return decorator
+
+
+def validate_with(schema):
+    """Wraps a get/post/put/delete method in a Flask-restful Resource, and
+    validates the request body with the given Voluptuous schema. If it passed,
+    sets `validated_body` on the request object, else abort(400) with helpful
+    error messages.
+    """
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            if not (request.json or request.data or request.form):
+                flask.ext.restful.abort(400, message='Validation error.',
+                                        errors=['No data provided.'])
+            try:
+                data = schema(prepare_request_data(request))
+            except voluptuous.MultipleInvalid as err:
+                flask.ext.restful.abort(400,
+                                        message='Validation error.',
+                                        errors=[str(e) for e in err.errors])
+            setattr(request, 'validated_body', data)
+            return f(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
 import projects, bams, runs, genotypes, tasks, comments

cycledash/api/bams.py

@@ -3,40 +3,67 @@
 from flask.ext.restful import abort, fields
 from sqlalchemy import select, desc
 import voluptuous
+from voluptuous import Schema, Required, Any, Exclusive, Coerce
 
 from common.helpers import tables, find
-from cycledash.validations import CreateBam, UpdateBam, expect_one_of
+from cycledash.validations import expect_one_of, PathString, Doc
 from cycledash import db
-from cycledash.helpers import validate_with, abort_if_none_for, marshal_with
+from cycledash.helpers import abort_if_none_for
+from cycledash.validations import Doc
 import workers.indexer
 
 import projects
-from . import Resource
-
-
-bam_fields = {
-    "id": fields.Integer,
-    "project_id": fields.Integer,
-    "name": fields.String,
-    "normal": fields.Boolean,
-    "notes": fields.String,
-    "resection_date": fields.String,
-    "tissues": fields.String,
-    "uri": fields.String
-}
+from . import Resource, marshal_with, validate_with
+
+
+CreateBam = Schema({
+    Required('uri'): PathString,
+
+    # One of `project` is required, but not supported in voluptuous, so we
+    # enforce this in code. cf. https://github.com/alecthomas/voluptuous/issues/115
+    Exclusive('project_id', 'project'): Coerce(int),
+    Exclusive('project_name', 'project'): unicode,
+
+    'name': unicode,
+    'notes': unicode,
+    'tissues': unicode,
+    'resection_date': unicode,
+})
+
+UpdateBam = Schema({
+    'name': unicode,
+    'notes': unicode,
+    'tissues': unicode,
+    'resection_date': unicode,
+    'uri': PathString
+})
+
+BamFields = Schema({
+    Doc('id', 'The internal ID of the BAM.'): long,
+    Doc('project_id', 'The internal ID of the project.'): long,
+    Doc('name', 'The name of the BAM.'): Any(basestring, None),
+    Doc('notes', 'Any notes or ancillary data.'): Any(basestring, None),
+    Doc('resection_date',
+        ('The date the tissue sample'
+         'for these reads was extracted')): Any(basestring, None),
+    Doc('normal',
+        'Whether or not the sample is from normal tissue.'): Any(bool, None),
+    Doc('tissues', 'Tissue type of sample.'): Any(basestring, None),
+    Doc('uri', 'The URI of the BAM on HDFS.'): PathString
+})
 
 
 class BamList(Resource):
     require_auth = True
-    @marshal_with(bam_fields, envelope='bams')
+    @marshal_with(BamFields, envelope='bams')
     def get(self):
         """Get list of all BAMs."""
         with tables(db.engine, 'bams') as (con, bams):
             q = select(bams.c).order_by(desc(bams.c.id))
             return [dict(r) for r in con.execute(q).fetchall()]
 
     @validate_with(CreateBam)
-    @marshal_with(bam_fields)
+    @marshal_with(BamFields)
     def post(self):
         """Create a new BAM.
 
@@ -60,15 +87,15 @@ def post(self):
 
 class Bam(Resource):
     require_auth = True
-    @marshal_with(bam_fields)
+    @marshal_with(BamFields)
     def get(self, bam_id):
         """Get a BAM by its ID."""
         with tables(db.engine, 'bams') as (con, bams):
             q = select(bams.c).where(bams.c.id == bam_id)
             return dict(_abort_if_none(q.execute().fetchone(), bam_id))
 
     @validate_with(UpdateBam)
-    @marshal_with(bam_fields)
+    @marshal_with(BamFields)
     def put(self, bam_id):
         """Update the BAM by its ID."""
         with tables(db.engine, 'bams') as (con, bams):
@@ -77,7 +104,7 @@ def put(self, bam_id):
             ).returning(*bams.c)
             return dict(_abort_if_none(q.execute().fetchone(), bam_id))
 
-    @marshal_with(bam_fields)
+    @marshal_with(BamFields)
     def delete(self, bam_id):
         """Delete a BAM by its ID."""
         with tables(db.engine, 'bams') as (con, bams):

cycledash/api/comments.py

@@ -3,35 +3,69 @@
 from flask import jsonify, request
 from flask.ext.restful import abort, fields
 from sqlalchemy import select, func, desc
+from voluptuous import Any, Required, Coerce, Schema
 
 from common.helpers import tables, to_epoch
 from cycledash import db
 from cycledash.helpers import (prepare_request_data, success_response,
-                               validate_with, abort_if_none_for, EpochField,
-                               marshal_with, camelcase_dict)
-from cycledash.validations import CreateComment, DeleteComment, UpdateComment
-
-from . import Resource
-
-
-comment_fields = {
-    "id": fields.Integer,
-    "vcf_id": fields.Integer,
-    "sample_name": fields.String,
-    "contig": fields.String,
-    "position": fields.Integer,
-    "reference": fields.String,
-    "alternates": fields.String,
-    "comment_text": fields.String,
-    "author_name": fields.String,
-    "created": EpochField,
-    "last_modified": EpochField
-}
+                               abort_if_none_for, camelcase_dict)
+from cycledash.validations import Doc, to_epoch
+
+from . import Resource, validate_with, marshal_with
+
+
+CreateComment = Schema({
+    Required("sample_name"): basestring,
+    Required("contig"): basestring,
+    Required("position"): Coerce(int),
+    Required("reference"): basestring,
+    Required("alternates"): basestring,
+    Required("comment_text"): basestring,
+    "author_name": basestring,
+})
+
+DeleteComment = Schema({
+    Required('last_modified'): Coerce(float),
+})
+
+UpdateComment = Schema({
+    Required('last_modified'): Coerce(float),
+    "comment_text": basestring,
+    "author_name": basestring,
+})
+
+CommentFields = Schema({
+    Doc('id', 'The internal ID of the Comment.'):
+        long,
+    Doc('vcf_id', 'The ID of the Run this comment is associated with.'):
+        long,
+    Doc('sample_name', 'The name of the sample this comment is on.'):
+        basestring,
+    Doc('contig', 'The contig of the variant this comment is on.'):
+        basestring,
+    Doc('position', 'The position of the variant this comment is on.'):
+        int,
+    Doc('reference', 'The reference of the variant this comment is on.'):
+        basestring,
+    Doc('alternates',
+        'The alternate allele of the variant this comment is on.'):
+        basestring,
+    Doc('comment_text', 'The text of the comment.'):
+        Any(basestring, None),
+    Doc('author_name', 'The name of the author of this comment.'):
+        Any(basestring, None),
+    Doc('created',
+        'The time at which the comment was created (in epoch time).'):
+        Coerce(to_epoch),
+    Doc('last_modified',
+        'The last modified time of the comment (in epoch time).'):
+        Coerce(to_epoch)
+})
 
 
 class CommentList(Resource):
     require_auth = True
-    @marshal_with(comment_fields, envelope='comments')
+    @marshal_with(CommentFields, envelope='comments')
     def get(self, run_id):
         """Get a list of all comments."""
         with tables(db.engine, 'user_comments') as (con, comments):
@@ -40,7 +74,7 @@ def get(self, run_id):
             return [dict(c) for c in q.execute().fetchall()]
 
     @validate_with(CreateComment)
-    @marshal_with(comment_fields)
+    @marshal_with(CommentFields)
     def post(self, run_id):
         """Create a comment."""
         with tables(db.engine, 'user_comments') as (con, comments):
@@ -53,14 +87,14 @@ def post(self, run_id):
 
 class Comment(Resource):
     require_auth = True
-    @marshal_with(comment_fields)
+    @marshal_with(CommentFields)
     def get(self, run_id, comment_id):
         """Get comment with the given ID."""
         with tables(db.engine, 'user_comments') as (con, comments):
             return _get_comment(comments, id=comment_id, vcf_id=run_id)
 
     @validate_with(UpdateComment)
-    @marshal_with(comment_fields)
+    @marshal_with(CommentFields)
     def put(self, run_id, comment_id):
         """Update the comment with the given ID.
 
@@ -80,7 +114,7 @@ def put(self, run_id, comment_id):
             return dict(q.execute().fetchone())
 
     @validate_with(DeleteComment)
-    @marshal_with(comment_fields)
+    @marshal_with(CommentFields)
     def delete(self, run_id, comment_id):
         """Delete the comment with the given ID.