Serve schemas

api/api.py

@@ -17,6 +17,7 @@
 from handlers import containerhandler
 from handlers import collectionshandler
 from handlers import searchhandler
+from handlers import schemahandler
 
 log = config.log
 
@@ -56,12 +57,14 @@ def get_js(self):
     # any character allowed except '/''
     'tag_re': '[^/]{3,24}',
     # filename regex
-    # length between 3 and 60 characters
     # any character allowed except '/'
     'filename_re': '[^/]+',
     # note id regex
     # hexadecimal string exactly of length 24
-    'note_id_re': '[0-9a-f]{24}'
+    'note_id_re': '[0-9a-f]{24}',
+    # schema regex
+    # example: schema_path/schema.json
+    'schema_re': '[^/.]{3,60}/[^/.]{3,60}\.json'
 }
 
 def _format(route):
@@ -134,7 +137,8 @@ def _format(route):
     webapp2.Route(_format(r'/api/<par_cont_name:groups>/<par_id:{group_id_re}>/<cont_name:projects>'),          containerhandler.ContainerHandler, name='cont_sublist_groups', handler_method='get_all', methods=['GET']),
     webapp2.Route(_format(r'/api/<par_cont_name:{cont_name_re}>/<par_id:{cid_re}>/<cont_name:{cont_name_re}>'), containerhandler.ContainerHandler, name='cont_sublist', handler_method='get_all', methods=['GET']),
     webapp2.Route(_format(r'/api/search'),                                            searchhandler.SearchHandler, name='es_proxy', methods=['GET']),
-    webapp2.Route(_format(r'/api/search/<cont_name:{cont_name_re}>'),                 searchhandler.SearchHandler, name='es_proxy', methods=['GET']),
+    webapp2.Route(_format(r'/api/search/<cont_name:{cont_name_re}>'),                 searchhandler.SearchHandler, name='es_proxy_1', methods=['GET']),
+    webapp2.Route(_format(r'/api/schemas/<schema:{schema_re}>'),                      schemahandler.SchemaHandler, name='schemas', methods=['GET']),
 ]
 
 

api/config.py

@@ -1,5 +1,6 @@
 import os
 import copy
+import glob
 import logging
 import pymongo
 import datetime
@@ -30,7 +31,7 @@
     'site': {
         'id': 'local',
         'name': 'Local',
-        'url': 'https://localhost/api',
+        'api_url': 'https://localhost/api',
         'central_url': 'https://sdmc.scitran.io/api',
         'registered': False,
         'ssl_cert': None,
@@ -46,8 +47,9 @@
         'db_connect_timeout': '2000',
         'db_server_selection_timeout': '3000',
         'data_path': os.path.join(os.path.dirname(__file__), '../persistent/data'),
+        'schema_path': 'api/schemas',
         'elasticsearch_host': 'localhost:9200',
-    }
+    },
 }
 
 __config = copy.deepcopy(DEFAULT_CONFIG)
@@ -87,6 +89,63 @@
 
 es = elasticsearch.Elasticsearch([__config['persistent']['elasticsearch_host']])
 
+# validate the lists of json schemas
+schema_path = __config['persistent']['schema_path']
+
+expected_mongo_schemas = set([
+    'acquisition.json',
+    'collection.json',
+    'container.json',
+    'file.json',
+    'group.json',
+    'note.json',
+    'permission.json',
+    'project.json',
+    'session.json',
+    'subject.json',
+    'user.json',
+    'avatars.json',
+    'tag.json'
+])
+expected_input_schemas = set([
+    'acquisition.json',
+    'collection.json',
+    'container.json',
+    'file.json',
+    'group.json',
+    'note.json',
+    'packfile.json',
+    'permission.json',
+    'project.json',
+    'session.json',
+    'subject.json',
+    'user.json',
+    'avatars.json',
+    'download.json',
+    'tag.json',
+    'enginemetadata.json',
+    'uploader.json',
+    'reaper.json'
+])
+mongo_schemas = set()
+input_schemas = set()
+
+# check that the lists of schemas are correct
+for schema_filepath in glob.glob(schema_path + '/mongo/*.json'):
+    schema_file = os.path.basename(schema_filepath)
+    mongo_schemas.add(schema_file)
+    with open(schema_filepath, 'rU') as f:
+        pass
+
+assert mongo_schemas == expected_mongo_schemas, '{} is different from {}'.format(mongo_schemas, expected_mongo_schemas)
+
+for schema_filepath in glob.glob(schema_path + '/input/*.json'):
+    schema_file = os.path.basename(schema_filepath)
+    input_schemas.add(schema_file)
+    with open(schema_filepath, 'rU') as f:
+        pass
+
+assert input_schemas == expected_input_schemas, '{} is different from {}'.format(input_schemas, expected_input_schemas)
 
 def initialize_db():
     log.info('Initializing database, creating indexes')
@@ -104,7 +163,7 @@ def initialize_db():
 
     now = datetime.datetime.utcnow()
     db.groups.update_one({'_id': 'unknown'}, {'$setOnInsert': { 'created': now, 'modified': now, 'name': 'Unknown', 'roles': []}}, upsert=True)
-    db.sites.replace_one({'_id': __config['site']['id']}, {'name': __config['site']['name'], 'site_url': __config['site']['url']}, upsert=True)
+    db.sites.replace_one({'_id': __config['site']['id']}, {'name': __config['site']['name'], 'site_url': __config['site']['api_url']}, upsert=True)
 
 
 def get_config():

api/download.py

@@ -198,7 +198,8 @@ def download(self):
                 config.db.projects.update_one({'_id': project_id}, {'$inc': {'counter': 1}})
         else:
             req_spec = self.request.json_body
-            validator = validators.payload_from_schema_file(self, 'download.json')
+            payload_schema_uri = util.schema_uri('input', 'download.json')
+            validator = validators.from_schema_path(payload_schema_uri)
             validator(req_spec, 'POST')
             log.debug(json.dumps(req_spec, sort_keys=True, indent=4, separators=(',', ': ')))
             return self._preflight_archivestream(req_spec)

api/handlers/containerhandler.py

@@ -306,8 +306,10 @@ def get_groups_with_project(self):
 
 
     def _get_validators(self):
-        mongo_validator = validators.mongo_from_schema_file(self.config.get('storage_schema_file'))
-        payload_validator = validators.payload_from_schema_file(self.config.get('payload_schema_file'))
+        mongo_schema_uri = util.schema_uri('mongo', self.config.get('storage_schema_file'))
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        payload_schema_uri = util.schema_uri('input', self.config.get('payload_schema_file'))
+        payload_validator = validators.from_schema_path(payload_schema_uri)
         return mongo_validator, payload_validator
 
     def _get_parent_container(self, payload):
@@ -329,7 +331,6 @@ def _get_parent_container(self, payload):
         log.debug(parent_container)
         return parent_container, parent_id_property
 
-
     def _get_container(self, _id):
         try:
             container = self.storage.get_container(_id)

api/handlers/grouphandler.py

@@ -1,6 +1,7 @@
 import datetime
 
 from .. import base
+from .. import util
 from .. import config
 from .. import debuginfo
 from .. import validators
@@ -58,8 +59,10 @@ def put(self, _id):
             self.abort(404, 'no such Group: ' + _id)
         permchecker = groupauth.default(self, group)
         payload = self.request.json_body
-        mongo_validator = validators.mongo_from_schema_file('group.json')
-        payload_validator = validators.payload_from_schema_file('group.json')
+        mongo_schema_uri = util.schema_uri('mongo', 'group.json')
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        payload_schema_uri = util.schema_uri('input', 'group.json')
+        payload_validator = validators.from_schema_path(payload_schema_uri)
         payload_validator(payload, 'PUT')
         result = mongo_validator(permchecker(self.storage.exec_op))('PUT', _id=_id, payload=payload)
         if result.modified_count == 1:
@@ -71,8 +74,10 @@ def post(self):
         self._init_storage()
         permchecker = groupauth.default(self, None)
         payload = self.request.json_body
-        mongo_validator = validators.mongo_from_schema_file('group.json')
-        payload_validator = validators.payload_from_schema_file('group.json')
+        mongo_schema_uri = util.schema_uri('mongo', 'group.json')
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        payload_schema_uri = util.schema_uri('input', 'group.json')
+        payload_validator = validators.from_schema_path(payload_schema_uri)
         payload_validator(payload, 'POST')
         payload['created'] = payload['modified'] = datetime.datetime.utcnow()
         payload['roles'] = [{'_id': self.uid, 'access': 'admin', 'site': self.user_site}] if self.uid else []

api/handlers/listhandler.py

@@ -183,9 +183,11 @@ def _initialize_request(self, cont_name, list_name, _id, query_params=None):
                 permchecker = permchecker(self, container)
         else:
             self.abort(404, 'Element {} not found in container {}'.format(_id, storage.cont_name))
-        mongo_validator = validators.mongo_from_schema_file(config.get('storage_schema_file'))
-        input_validator = validators.payload_from_schema_file(config.get('input_schema_file'))
-        keycheck = validators.key_check(config.get('storage_schema_file'))
+        mongo_schema_uri = util.schema_uri('mongo', config.get('storage_schema_file'))
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        input_schema_uri = util.schema_uri('input', config.get('input_schema_file'))
+        input_validator = validators.from_schema_path(input_schema_uri)
+        keycheck = validators.key_check(mongo_schema_uri)
         return container, permchecker, storage, mongo_validator, input_validator, keycheck
 
 

api/handlers/schemahandler.py

@@ -0,0 +1,21 @@
+import os
+import json
+import datetime
+
+from .. import base
+from .. import config
+
+log = config.log
+
+class SchemaHandler(base.RequestHandler):
+
+    def __init__(self, request=None, response=None):
+        super(SchemaHandler, self).__init__(request, response)
+
+    def get(self, schema, **kwargs):
+        schema_path = os.path.join(config.get_item('persistent', 'schema_path'), schema)
+        try:
+            with open(schema_path, 'ru') as f:
+                return json.load(f)
+        except IOError as e:
+            self.abort(404, str(e))

api/handlers/userhandler.py

@@ -3,6 +3,7 @@
 import requests
 
 from .. import base
+from .. import util
 from .. import config
 from .. import validators
 from ..auth import userauth, always_ok, ROLES
@@ -64,8 +65,10 @@ def put(self, _id):
         user = self._get_user(_id)
         permchecker = userauth.default(self, user)
         payload = self.request.json_body
-        mongo_validator = validators.mongo_from_schema_file('user.json')
-        payload_validator = validators.payload_from_schema_file('user.json')
+        mongo_schema_uri = util.schema_uri('mongo', 'user.json')
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        payload_schema_uri = util.schema_uri('input', 'user.json')
+        payload_validator = validators.from_schema_path(payload_schema_uri)
         payload_validator(payload, 'PUT')
         payload['modified'] = datetime.datetime.utcnow()
         result = mongo_validator(permchecker(self.storage.exec_op))('PUT', _id=_id, payload=payload)
@@ -78,8 +81,10 @@ def post(self):
         self._init_storage()
         permchecker = userauth.default(self)
         payload = self.request.json_body
-        mongo_validator = validators.mongo_from_schema_file('user.json')
-        payload_validator = validators.payload_from_schema_file('user.json')
+        mongo_schema_uri = util.schema_uri('mongo', 'user.json')
+        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
+        payload_schema_uri = util.schema_uri('input', 'user.json')
+        payload_validator = validators.from_schema_path(payload_schema_uri)
         payload_validator(payload, 'POST')
         payload['created'] = payload['modified'] = datetime.datetime.utcnow()
         payload['root'] = payload.get('root', False)

api/root.py

@@ -2,6 +2,9 @@
 import markdown
 
 from . import base
+from . import config
+
+log = config.log
 
 class Root(base.RequestHandler):
 
@@ -80,4 +83,4 @@ def get(self):
             self.response.write('</form>\n')
         self.response.write(markdown.markdown(resources, ['extra']))
         self.response.write('</body>\n')
-        self.response.write('</html>\n')
+        self.response.write('</html>\n')

api/upload.py

@@ -170,7 +170,8 @@ def uploader(self):
                 self.abort(400, str(e))
             if not file_store.metadata:
                 self.abort(400, 'metadata is missing')
-            metadata_validator = validators.payload_from_schema_file('uploader.json')
+            payload_schema_uri = util.schema_uri('input', 'uploader.json')
+            metadata_validator = validators.from_schema_path(payload_schema_uri)
             metadata_validator(file_store.metadata, 'POST')
             try:
                 target_containers = reaperutil.create_root_to_leaf_hierarchy(file_store.metadata, file_store.files)
@@ -216,7 +217,8 @@ def engine(self):
                 self.abort(400, str(e))
             if not file_store.metadata:
                 self.abort(400, 'metadata is missing')
-            metadata_validator = validators.payload_from_schema_file('enginemetadata.json')
+            payload_schema_uri = util.schema_uri('input', 'enginemetadata.json')
+            metadata_validator = validators.from_schema_path(payload_schema_uri)
             metadata_validator(file_store.metadata, 'POST')
             file_infos = file_store.metadata['acquisition'].pop('files', [])
             now = datetime.datetime.utcnow()

api/util.py

@@ -8,6 +8,7 @@
 import tempdir as tempfile
 import enum as baseEnum
 
+from . import config
 MIMETYPES = [
     ('.bvec', 'text', 'bvec'),
     ('.bval', 'text', 'bval'),
@@ -136,6 +137,13 @@ def send_json_http_exception(response, message, code):
     response.headers['Content-Type'] = 'application/json; charset=utf-8'
     response.write(content)
 
+def schema_uri(type_, schema_name):
+    return '/'.join([
+        config.get_item('site', 'api_url'),
+        'schemas',
+        type_, schema_name
+    ])
+
 class Enum(baseEnum.Enum):
     # Enum strings are prefixed by their class: "Category.classifier".
     # This overrides that behaviour and removes the prefix.