Merge pull request #816 from neuroscout/enh/marshmallow3

Update to Marshamallow 3, also upgrade webargs flask_apispec

neuroscout/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6-stretch
+FROM python:3.6-buster
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN mkdir -p /usr/src/app
@@ -21,6 +21,7 @@ RUN npm install -g yarn
 
 COPY requirements.txt /usr/src/app/
 COPY optional_requirements.txt /usr/src/app
+RUN pip install setuptools==45
 RUN pip install --no-cache-dir -r requirements.txt
 RUN pip install --no-cache-dir -r optional_requirements.txt
 RUN python -m pliers.support.download

neuroscout/api_spec.py

@@ -3,7 +3,7 @@
 from .auth import add_auth_to_swagger
 from apispec import APISpec
 from flask_apispec.extension import FlaskApiSpec
-import webargs as wa
+from webargs import fields
 from apispec.ext.marshmallow import MarshmallowPlugin
 
 file_plugin = MarshmallowPlugin()
@@ -21,5 +21,5 @@
 
 
 @file_plugin.map_to_openapi_type('file', None)
-class FileField(wa.fields.Raw):
+class FileField(fields.Raw):
     pass

neuroscout/models/dataset.py

@@ -47,6 +47,7 @@ def mean_age(self):
         else:
             return None
 
+    @hybrid_property
     def percent_female(self):
         values = GroupPredictorValue.query.join(GroupPredictor).filter_by(
             dataset_id=self.id).filter(

neuroscout/requirements.txt

@@ -1,19 +1,20 @@
 altair
 alembic==1.3.1
-apispec==1.1.0
+apispec==3.3.2
 appdirs==1.4.3
 blinker==1.4
 celery
 citeproc-py==0.4.0
 citeproc-py-styles==0.1.1
 datalad==0.10.2
+dataclasses==0.7
 decorator==4.0.11
 docopt==0.6.2
 duecredit==0.6.1
 pybids==0.10.1
--e git+https://github.com/neurolearn/pynv#egg=pynv
+pynv==0.2
 Flask==1.1.1
-flask-apispec==0.8.0
+flask-apispec==0.10.0
 Flask-Caching==1.3.3
 Flask-cors==3.0.2
 Flask-JWT==0.3.2
@@ -35,7 +36,7 @@ ipython==6.2.1
 itsdangerous==0.24
 Jinja2==2.10.1
 MarkupSafe==1.0
-marshmallow==2.19.1
+marshmallow==3.7.1
 moviepy==0.2.3.5
 nibabel==2.1.0
 nltk==3.2.2
@@ -64,7 +65,7 @@ soundfile
 pytesseract
 SQLAlchemy==1.3.0
 tqdm==4.11.2
-webargs>=0.18.0,<6.0.0
+webargs==6.1.1
 Werkzeug==0.15.3
 WTForms==2.1
 xlrd

neuroscout/resources/analysis/endpoints.py

@@ -5,7 +5,7 @@
 from ...database import db
 from os.path import exists
 import datetime
-import webargs as wa
+from webargs import fields
 import json
 
 from ...utils.db import put_record
@@ -25,16 +25,17 @@ class AnalysisMethodResource(MethodResource):
 
 class AnalysisRootResource(AnalysisMethodResource):
     """" Resource for root address """
-    @marshal_with(AnalysisSchema(many=True,
-                                 only=['hash_id', 'name', 'description', 'status',
-                                       'dataset_id', 'modified_at', 'user']))
+    @marshal_with(AnalysisSchema(
+        many=True,
+        only=['hash_id', 'name', 'description', 'status',
+              'dataset_id', 'modified_at', 'user']))
     @doc(summary='Returns list of public analyses.')
     @use_kwargs({
-        'name': wa.fields.DelimitedList(
-            wa.fields.Str(), description="Analysis name(s)"),
-        'dataset_id': wa.fields.DelimitedList(
-            wa.fields.Number(), description="Dataset id(s)")
-        }, locations=['query'])
+        'name': fields.DelimitedList(
+            fields.Str(), description="Analysis name(s)"),
+        'dataset_id': fields.DelimitedList(
+            fields.Number(), description="Dataset id(s)")
+        }, location='query')
     def get(self, **kwargs):
         query = Analysis.query.filter_by(private=False, status='PASSED')
         for param in kwargs:
@@ -92,12 +93,12 @@ def delete(self, analysis):
 
 class AnalysisFillResource(AnalysisMethodResource):
     @use_kwargs({
-        'partial': wa.fields.Boolean(
+        'partial': fields.Boolean(
             description='Attempt partial fill if complete is not possible.',
             missing=True),
-        'dryrun': wa.fields.Boolean(
+        'dryrun': fields.Boolean(
             description="Don't commit autofill results to database")
-        }, locations=['query'])
+        }, location='query')
     @doc(summary='Auto fill fields from model.')
     @owner_required
     def post(self, analysis, partial=True, dryrun=False):

neuroscout/resources/analysis/reports.py

@@ -1,6 +1,7 @@
 import datetime
 from hashids import Hashids
-import webargs as wa
+from webargs import fields
+from marshmallow import INCLUDE, Schema
 from flask_apispec import doc, use_kwargs, MethodResource, marshal_with
 from flask import current_app
 from pathlib import Path
@@ -27,9 +28,9 @@ def _validation_hash(analysis_id):
 @marshal_with(AnalysisCompiledSchema)
 class CompileAnalysisResource(MethodResource):
     @use_kwargs({
-        'build': wa.fields.Boolean(
+        'build': fields.Boolean(
             description='Build Analysis object')
-        }, locations=['query'])
+        }, location='query')
     @doc(summary='Compile and lock analysis.')
     @owner_required
     def post(self, analysis, build=False):
@@ -59,11 +60,11 @@ def get(self, analysis):
 
 @marshal_with(ReportSchema)
 @use_kwargs({
-    'run_id': wa.fields.DelimitedList(wa.fields.Int(),
-                                      description='Run id(s).'),
-    'sampling_rate': wa.fields.Number(description='Sampling rate in Hz'),
-    'scale': wa.fields.Boolean(description='Scale columns for plotting'),
-}, locations=['query'])
+    'run_id': fields.DelimitedList(
+        fields.Int(), description='Run id(s).'),
+    'sampling_rate': fields.Number(description='Sampling rate in Hz'),
+    'scale': fields.Boolean(description='Scale columns for plotting'),
+}, location='query')
 @doc(tags=['analysis'])
 class ReportResource(MethodResource):
     @doc(summary='Generate analysis reports.')
@@ -166,19 +167,31 @@ def _create_collection(analysis, force=False):
     return upload
 
 
+class FormSchema(Schema):
+    validation_hash = fields.Str(required=True)
+    collection_id = fields.Int()
+    force = fields.Bool()
+    level = fields.Str()
+    n_subjects = fields.Number()
+
+    class Meta:
+        unknown = INCLUDE
+
+
+class FileSchema(Schema):
+    image_file = FileField(required=True)
+
+    class Meta:
+        unknown = INCLUDE
+
+
 @doc(tags=['analysis'])
 class AnalysisUploadResource(MethodResource):
     @doc(summary='Upload fitlins analysis results. ',
          consumes=['multipart/form-data', 'application/x-www-form-urlencoded'])
     @marshal_with(NeurovaultCollectionSchemaStatus)
-    @use_kwargs({
-        "validation_hash": wa.fields.Str(required=True),
-        "image_file": FileField(required=False),
-        "collection_id": wa.fields.Int(required=False),
-        "force": wa.fields.Bool(),
-        "level": wa.fields.Str(),
-        "n_subjects": wa.fields.Number(description='Number of subjects'),
-        }, locations=["files", "form"])
+    @use_kwargs(FormSchema, location="form")
+    @use_kwargs(FileSchema, location="files")
     @fetch_analysis
     def post(self, analysis, validation_hash, collection_id=None,
              image_file=None, level=None, n_subjects=None, force=False):

neuroscout/resources/dataset.py

@@ -1,5 +1,5 @@
 from flask_apispec import MethodResource, marshal_with, doc, use_kwargs
-import webargs as wa
+from webargs import fields
 from ..models import Dataset
 from ..core import cache
 from .utils import first_or_404
@@ -17,10 +17,10 @@ def get(self, dataset_id):
 class DatasetListResource(MethodResource):
     @doc(tags=['dataset'], summary='Returns list of datasets.')
     @use_kwargs({
-        'active_only': wa.fields.Boolean(
+        'active_only': fields.Boolean(
             missing=True, description="Return only active Datasets")
         },
-        locations=['query'])
+        location='query')
     @cache.cached(60 * 60 * 24 * 300, query_string=True)
     @marshal_with(DatasetSchema(
         many=True, exclude=['dataset_address', 'preproc_address']))

neuroscout/resources/predictor.py

@@ -1,4 +1,4 @@
-import webargs as wa
+from webargs import fields
 import tempfile
 import json
 from sqlalchemy import func
@@ -59,18 +59,18 @@ def get_predictors(newest=True, active=True, user=None, **kwargs):
 class PredictorListResource(MethodResource):
     @doc(tags=['predictors'], summary='Get list of predictors.',)
     @use_kwargs({
-        'run_id': wa.fields.DelimitedList(
-            wa.fields.Int(), description="Run id(s). Warning, slow query."),
-        'name': wa.fields.DelimitedList(wa.fields.Str(),
-                                        description="Predictor name(s)"),
-        'active_only': wa.fields.Boolean(
+        'run_id': fields.DelimitedList(
+            fields.Int(), description="Run id(s). Warning, slow query."),
+        'name': fields.DelimitedList(
+            fields.Str(), description="Predictor name(s)"),
+        'active_only': fields.Boolean(
             missing=True,
             description="Return only active Predictors"),
-        'newest': wa.fields.Boolean(
+        'newest': fields.Boolean(
             missing=True,
             description="Return only newest Predictor by name")
         },
-        locations=['query'])
+        location='query')
     @cache.cached(60 * 60 * 24 * 300, query_string=True)
     @marshal_with(PredictorSchema(many=True))
     def get(self, **kwargs):
@@ -116,22 +116,22 @@ class PredictorCollectionResource(MethodResource):
          consumes=['multipart/form-data', 'application/x-www-form-urlencoded'])
     @marshal_with(PredictorCollectionSchema)
     @use_kwargs({
-        "collection_name": wa.fields.Str(
+        "collection_name": fields.Str(
             required=True,
             description="Name of collection"
         ),
-        "event_files": wa.fields.List(
+        "event_files": fields.List(
             FileField(), required=True,
             description="TSV files with additional Predictors to be created.\
             Required columns: onset, duration, any number of columns\
             with values for new Predictors."),
-        "runs": wa.fields.List(
-            wa.fields.DelimitedList(wa.fields.Int()),
+        "runs": fields.List(
+            fields.DelimitedList(fields.Int()),
             required=True
             ),
-        "dataset_id": wa.fields.Int(required=True, description="Dataset id."),
-        "descriptions": wa.fields.Str(description="Column descriptions")
-        }, locations=["files", "form"])
+        "dataset_id": fields.Int(required=True, description="Dataset id."),
+        "descriptions": fields.Str(description="Column descriptions")
+        }, location="form")
     @auth_required
     def post(self, collection_name, event_files, runs, dataset_id,
              descriptions=None):
@@ -155,9 +155,9 @@ def post(self, collection_name, event_files, runs, dataset_id,
 
     @doc(summary='Get predictor collection by id.')
     @use_kwargs(
-        {'collection_id': wa.fields.Int(description="Predictor Collection id.",
+        {'collection_id': fields.Int(description="Predictor Collection id.",
                                         required=True)},
-        locations=['query'])
+        location='query')
     @marshal_with(PredictorCollectionSchema)
     def get(self, collection_id):
         return first_or_404(
@@ -168,17 +168,17 @@ class PredictorEventListResource(MethodResource):
     @doc(tags=['predictors'], summary='Get events for predictor(s)',)
     @marshal_with(PredictorEventSchema(many=True))
     @use_kwargs({
-        'run_id': wa.fields.DelimitedList(
-            wa.fields.Int(),
+        'run_id': fields.DelimitedList(
+            fields.Int(),
             description="Run id(s)"),
-        'predictor_id': wa.fields.DelimitedList(
-            wa.fields.Int(),
+        'predictor_id': fields.DelimitedList(
+            fields.Int(),
             description="Predictor id(s)",
             required=True),
-        'stimulus_timing': wa.fields.Boolean(
+        'stimulus_timing': fields.Boolean(
             missing=False,
             description="Return stimulus timing and information")
-        }, locations=['query'])
+        }, location='query')
     @cache.cached(60 * 60 * 24 * 300, query_string=True)
     def get(self, predictor_id, run_id=None, stimulus_timing=False):
         return dump_predictor_events(

neuroscout/resources/run.py

@@ -1,5 +1,5 @@
 from flask_apispec import MethodResource, marshal_with, use_kwargs, doc
-import webargs as wa
+from webargs import fields
 from ..models import Run
 from .utils import first_or_404
 from ..schemas.run import RunSchema
@@ -15,16 +15,16 @@ def get(self, run_id):
 class RunListResource(MethodResource):
     @doc(tags=['run'], summary='Returns list of runs.')
     @use_kwargs({
-        'session': wa.fields.DelimitedList(
-            wa.fields.Str(), description='Session number(s).'),
-        'number': wa.fields.DelimitedList(
-            wa.fields.Int(), description='Run number(s).'),
-        'task_id': wa.fields.DelimitedList(
-            wa.fields.Int(), description='Task id(s).'),
-        'subject': wa.fields.DelimitedList(
-            wa.fields.Str(), description='Subject id(s).'),
-        'dataset_id': wa.fields.Int(description='Dataset id.'),
-    }, locations=['query'])
+        'session': fields.DelimitedList(
+            fields.Str(), description='Session number(s).'),
+        'number': fields.DelimitedList(
+            fields.Int(), description='Run number(s).'),
+        'task_id': fields.DelimitedList(
+            fields.Int(), description='Task id(s).'),
+        'subject': fields.DelimitedList(
+            fields.Str(), description='Subject id(s).'),
+        'dataset_id': fields.Int(description='Dataset id.'),
+    }, location='query')
     @marshal_with(RunSchema(many=True))
     def get(self, **kwargs):
         try:

neuroscout/resources/task.py

@@ -1,6 +1,6 @@
 from flask_apispec import MethodResource, marshal_with, doc, use_kwargs
 from ..models import Task
-import webargs as wa
+from webargs import fields
 from .utils import first_or_404
 from ..schemas.task import TaskSchema
 
@@ -15,8 +15,8 @@ def get(self, task_id):
 class TaskListResource(MethodResource):
     @doc(tags=['run'], summary='Returns list of tasks.')
     @use_kwargs({
-        'dataset_id': wa.fields.Int(description='Dataset id(s).'),
-    }, locations=['query'])
+        'dataset_id': fields.Int(description='Dataset id(s).'),
+    }, location='query')
     @marshal_with(TaskSchema(many=True))
     def get(self, **kwargs):
         query = Task.query