Custom PHI lists at site/project level

api/api.py

@@ -9,13 +9,14 @@
 from .handlers.devicehandler        import DeviceHandler
 from .handlers.grouphandler         import GroupHandler
 from .handlers.listhandler          import FileListHandler, NotesListHandler, PermissionsListHandler, TagsListHandler
+from .handlers.projectsettings      import ProjectSettings, RulesHandler, RuleHandler
 from .handlers.refererhandler       import AnalysesHandler
 from .handlers.reporthandler        import ReportHandler
 from .handlers.resolvehandler       import ResolveHandler
 from .handlers.roothandler          import RootHandler
 from .handlers.schemahandler        import SchemaHandler
 from .handlers.userhandler          import UserHandler
-from .jobs.handlers                 import BatchHandler, JobsHandler, JobHandler, GearsHandler, GearHandler, RulesHandler, RuleHandler
+from .jobs.handlers                 import BatchHandler, JobsHandler, JobHandler, GearsHandler, GearHandler
 from .upload                        import Upload
 from .web.base                      import RequestHandler
 from . import config
@@ -201,8 +202,10 @@ def prefix(path, routes):
         prefix('/projects', [
             route('/groups',               ContainerHandler, h='get_groups_with_project',      m=['GET']),
             route('/recalc',               ContainerHandler, h='calculate_project_compliance', m=['POST']),
-            route('/<cid:{cid}>/template', ContainerHandler, h='set_project_template',         m=['POST']),
-            route('/<cid:{cid}>/template', ContainerHandler, h='delete_project_template',      m=['DELETE']),
+            route('/<cid:{cid}|site>/phi',     ProjectSettings,  h='get_phi',                      m=['GET']),
+            route('/<cid:{cid}|site>/phi',     ProjectSettings,  h='update_phi',                   m=['POST', 'PUT']),
+            route('/<cid:{cid}>/template', ProjectSettings,  h='set_project_template',         m=['POST']),
+            route('/<cid:{cid}>/template', ProjectSettings,  h='delete_project_template',      m=['DELETE']),
             route('/<cid:{cid}>/recalc',   ContainerHandler, h='calculate_project_compliance', m=['POST']),
             route('/<cid:{cid}>/rules',    RulesHandler,                                       m=['GET', 'POST']),
             route('/<cid:{cid}>/rules/<rid:{cid}>',  RuleHandler,                              m=['GET', 'PUT', 'DELETE']),

api/auth/listauth.py

@@ -6,7 +6,7 @@
 import sys
 
 from .. import config
-from . import _get_access, INTEGER_PERMISSIONS
+from . import _get_access, check_phi, INTEGER_PERMISSIONS
 
 log = config.log
 
@@ -19,7 +19,8 @@ def default_sublist(handler, container):
     """
     access = _get_access(handler.uid, container)
     def g(exec_op):
-        def f(method, _id, query_params=None, payload=None, exclude_params=None):
+        def f(method, _id, query_params=None, payload=None, exclude_params=None, phi=False):
+            log.debug(method)
             if method == 'GET' and container.get('public', False):
                 min_access = -1
             elif method == 'GET':
@@ -28,7 +29,10 @@ def f(method, _id, query_params=None, payload=None, exclude_params=None):
                 min_access = INTEGER_PERMISSIONS['rw']
             else:
                 min_access = sys.maxint
-
+            if method == 'GET':
+                log.debug(container)
+                if (not check_phi(handler.uid, container)) and phi:
+                    handler.abort(403, "User not authorized to view PHI fields on the container.")
             if access >= min_access:
                 return exec_op(method, _id, query_params, payload, exclude_params)
             else:
@@ -58,7 +62,11 @@ def group_tags_sublist(handler, container):
     """
     access = _get_access(handler.uid, container)
     def g(exec_op):
-        def f(method, _id, query_params = None, payload = None, exclude_params=None):
+        def f(method, _id, query_params = None, payload = None, exclude_params=None, phi=False):
+            if method == 'GET':
+                log.debug(container)
+                if (not check_phi(handler.uid, container)) and phi:
+                    handler.abort(403, "User not authorized to view PHI fields on the container.")
             if method == 'GET'  and access >= INTEGER_PERMISSIONS['ro']:
                 return exec_op(method, _id, query_params, payload, exclude_params)
             elif access >= INTEGER_PERMISSIONS['rw']:
@@ -90,17 +98,24 @@ def notes_sublist(handler, container):
     """
     access = _get_access(handler.uid, container)
     def g(exec_op):
-        def f(method, _id, query_params = None, payload = None, exclude_params=None):
+        def f(method, _id, query_params = None, payload = None, exclude_params=None, phi=False):
             if access >= INTEGER_PERMISSIONS['admin']:
                 pass
             elif method == 'POST' and access >= INTEGER_PERMISSIONS['rw'] and payload['user'] == handler.uid:
                 pass
             elif method == 'GET' and (access >= INTEGER_PERMISSIONS['ro'] or container.get('public')):
-                pass
+                if not check_phi(handler.uid, container) and phi:
+                    handler.abort(403, "User not authorized to view PHI fields on the container.")
             elif method in ['GET', 'DELETE', 'PUT'] and container['notes'][0]['user'] == handler.uid:
                 pass
             else:
                 handler.abort(403, 'user not authorized to perform a {} operation on the list'.format(method))
+
+            # Check phi access if GET and notes is considered phi by the project or site
+            if method == 'GET':
+                log.debug(container)
+                if (not check_phi(handler.uid, container)) and phi:
+                    handler.abort(403, "User not authorized to view PHI fields on the container.")
             return exec_op(method, _id, query_params, payload, exclude_params)
         return f
     return g

api/config.py

@@ -158,10 +158,12 @@ def apply_env_variables(config):
     'collection.json',
     'collection-update.json',
     'container.json',
+    'custom-phi-update.json',
     'device.json',
     'file.json',
     'file-update.json',
     'group-new.json',
+    "group-permission.json",
     'group-update.json',
     'info_update.json',
     'note.json',

api/dao/containerstorage.py

@@ -96,6 +96,16 @@ def recalc_sessions_compliance(self, project_id=None):
                         changed_sessions.append(s['_id'])
         return changed_sessions
 
+    def get_phi_fields(self, cid, projection=None):
+        log.debug(cid)
+        phi = config.db.project_phi.find_one({"project_id": str(cid)}, projection=projection)
+        if phi == None:
+            return {"fields":[]}
+        else:
+            return phi
+
+    def add_phi_fields(self, cid, update):
+        return config.db.project_phi.update({"project_id": cid}, {"$set": update}, upsert=True)
 
 class SessionStorage(ContainerStorage):
 

api/dao/liststorage.py

@@ -46,7 +46,7 @@ def get_container(self, _id, query_params=None):
             projection = {self.list_name + '.$': 1, 'permissions': 1, 'public': 1}
         return self.dbc.find_one(query, projection)
 
-    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None):
+    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None, phi=False): # pylint: disable=unused-argument
         """
         Generic method to exec an operation.
         The request is dispatched to the corresponding private methods.
@@ -220,7 +220,7 @@ def get_container(self, _id, query_params=None):
         projection = {self.list_name : 1, 'permissions': 1, 'public': 1}
         return self.dbc.find_one(query, projection)
 
-    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None):
+    def exec_op(self, action, _id=None, query_params=None, payload=None, exclude_params=None, phi=False): # pylint: disable=unused-argument
         """
         This method "flattens" the query parameter and the payload to handle string lists
         """

api/handlers/collectionshandler.py

@@ -9,6 +9,7 @@
 from ..web.request import AccessType
 
 from .containerhandler import ContainerHandler
+from .projectsettings import get_phi_fields
 
 log = config.log
 
@@ -32,6 +33,7 @@ def __init__(self, request=None, response=None):
         self.storage = self.container_handler_configurations['collections']['storage']
 
     def get(self, **kwargs):
+        log.debug(kwargs)
         return super(CollectionsHandler, self).get(cont_name='collections', **kwargs)
 
     def post(self):
@@ -166,16 +168,14 @@ def get_sessions(self, cid):
             permchecker = containerauth.list_public_request
         else:
             permchecker = containerauth.list_permission_checker(self)
-
-        projection = self.PHI_FIELDS.copy()
+        projection = get_phi_fields("sessions", "site")
         if self.is_true('phi'):
             projection = None
             phi = True
         else:
             phi = False
 
         sessions = permchecker(containerstorage.SessionStorage().exec_op)('GET', query=query, public=self.public_request, projection=projection, phi=phi)
-        log.debug(sessions)
         self._filter_all_permissions(sessions, self.uid)
         if self.is_true('measurements'):
             self._add_session_measurements(sessions)
@@ -210,8 +210,7 @@ def get_acquisitions(self, cid):
             permchecker = containerauth.list_public_request
         else:
             permchecker = containerauth.list_permission_checker(self)
-
-        projection = self.PHI_FIELDS.copy()
+        projection = get_phi_fields("acquisitions", "site")
         if self.is_true('phi'):
             projection = None
             phi = True

api/handlers/containerhandler.py

@@ -15,6 +15,7 @@
 from ..web import base
 from ..web.errors import APIStorageException
 from ..web.request import log_access, AccessType
+from .projectsettings import phi_payload, get_phi_fields, check_phi_enabled
 
 log = config.log
 
@@ -43,10 +44,6 @@ class ContainerHandler(base.RequestHandler):
         'acquisitions': True
     }
 
-    # Hard-coded PHI fields, will be changed to user set PHI fields
-    PHI_FIELDS = {'info': 0, 'subject.firstname':0, 'subject.lastname': 0, 'subject.sex': 0,
-                  'subject.age': 0, 'subject.race': 0, 'subject.ethnicity': 0, 'subject.info': 0, 'tags': 0, 'files.info':0}
-
     # This configurations are used by the ContainerHandler class to load the storage,
     # the permissions checker and the json schema validators used to handle a request.
     #
@@ -92,14 +89,16 @@ def __init__(self, request=None, response=None):
     @log_access(AccessType.view_container)
     def get(self, cont_name, **kwargs):
         _id = kwargs.get('cid')
-        projection = self.PHI_FIELDS.copy()
-        log.debug(self.PHI_FIELDS)
-        log.debug(projection)
         self.config = self.container_handler_configurations[cont_name]
         self.storage = self.config['storage']
+        projection = get_phi_fields(cont_name, _id)
         container = self._get_container(_id)
         log.debug(container)
-        if check_phi(self.uid, container) or self.superuser_request:
+        self.phi = False
+        if not check_phi_enabled(cont_name, _id):
+            self.phi = False
+            projection = None
+        elif check_phi(self.uid, container) or self.superuser_request:
             log.debug("PHI")
             log.debug(self.superuser_request)
             self.phi = True
@@ -111,6 +110,8 @@ def get(self, cont_name, **kwargs):
             result = permchecker(self.storage.exec_op)('GET', _id, projection=projection, phi=self.phi)
         except APIStorageException as e:
             self.abort(400, e.message)
+        if not check_phi_enabled(cont_name, _id):
+            self.phi = True
         if result is None:
             self.abort(404, 'Element not found in container {} {}'.format(self.storage.cont_name, _id))
         if not self.superuser_request and not self.is_true('join_avatars'):
@@ -123,7 +124,7 @@ def get(self, cont_name, **kwargs):
                 fileinfo['path'] = util.path_from_hash(fileinfo['hash'])
 
         inflate_job_info = cont_name == 'sessions'
-        result['analyses'] = AnalysisStorage().get_analyses(cont_name, _id, inflate_job_info, self.PHI_FIELDS.copy())
+        result['analyses'] = AnalysisStorage().get_analyses(cont_name, _id, inflate_job_info, projection)
         return self.handle_origin(result)
 
     def handle_origin(self, result):
@@ -328,9 +329,9 @@ def get_all(self, cont_name, par_cont_name=None, par_id=None):
         else:
             phi = False
             if projection == None:
-                projection = self.PHI_FIELDS.copy()
+                projection = get_phi_fields(cont_name, "site")
             else:
-                projection.update(self.PHI_FIELDS)
+                projection.update(get_phi_fields(cont_name, "site"))
         # select which permission filter will be applied to the list of results.
         if self.superuser_request:
             permchecker = always_ok
@@ -408,8 +409,7 @@ def get_all_for_user(self, cont_name, uid):
         if self.is_true('phi'):
             phi = True
         else:
-            projection = self.PHI_FIELDS.copy()
-
+            projection = get_phi_fields(cont_name, "site")
         # select which permission filter will be applied to the list of results.
         if self.superuser_request or self.user_is_admin:
             permchecker = always_ok
@@ -433,6 +433,7 @@ def get_all_for_user(self, cont_name, uid):
         self._filter_all_permissions(results, uid)
         return results
 
+    @phi_payload(method="POST")
     def post(self, cont_name):
         self.config = self.container_handler_configurations[cont_name]
         self.storage = self.config['storage']
@@ -472,6 +473,7 @@ def post(self, cont_name):
         else:
             self.abort(404, 'Element not added in container {}'.format(self.storage.cont_name))
 
+    @phi_payload(method="PUT")
     @validators.verify_payload_exists
     def put(self, cont_name, **kwargs):
         _id = kwargs.pop('cid')
@@ -585,35 +587,6 @@ def get_groups_with_project(self):
         group_ids = list(set((p['group'] for p in self.get_all('projects'))))
         return list(config.db.groups.find({'_id': {'$in': group_ids}}, ['label']))
 
-    def set_project_template(self, **kwargs):
-        project_id = kwargs.pop('cid')
-        self.config = self.container_handler_configurations['projects']
-        self.storage = self.config['storage']
-        container = self._get_container(project_id)
-
-        template = self.request.json_body
-        validators.validate_data(template, 'project-template.json', 'input', 'POST')
-        payload = {'template': template}
-        payload['modified'] = datetime.datetime.utcnow()
-
-        permchecker = self._get_permchecker(container)
-        result = permchecker(self.storage.exec_op)('PUT', _id=project_id, payload=payload)
-        return {'modified': result.modified_count}
-
-    def delete_project_template(self, **kwargs):
-        project_id = kwargs.pop('cid')
-        self.config = self.container_handler_configurations['projects']
-        self.storage = self.config['storage']
-        container = self._get_container(project_id)
-
-        payload = {'modified': datetime.datetime.utcnow()}
-        unset_payload = {'template': ''}
-
-        permchecker = self._get_permchecker(container)
-        result = permchecker(self.storage.exec_op)('PUT', _id=project_id, payload=payload, unset_payload=unset_payload)
-        return {'modified': result.modified_count}
-
-
     def calculate_project_compliance(self, **kwargs):
         project_id = kwargs.pop('cid', None)
         log.debug("project_id is {}".format(project_id))
@@ -659,6 +632,7 @@ def _get_parent_container(self, payload):
 
     def _get_container(self, _id, projection=None, get_children=False):
         try:
+            log.debug(self.storage)
             container = self.storage.get_container(_id, projection=projection, get_children=get_children)
         except APIStorageException as e:
             self.abort(400, e.message)

api/handlers/listhandler.py

@@ -17,8 +17,11 @@
 from ..dao import liststorage
 from ..dao import containerutil
 from ..web.errors import APIStorageException
+from ..dao.containerstorage import ProjectStorage
+from ..handlers.projectsettings import get_project_id, check_phi_enabled
 from ..web.request import log_access, AccessType
 
+log = config.log
 
 def initialize_list_configurations():
     """
@@ -67,7 +70,7 @@ def initialize_list_configurations():
                 'use_object_id': False,
                 'get_full_container': True,
                 'storage_schema_file': 'permission.json',
-                'input_schema_file': 'permission.json'
+                'input_schema_file': 'group-permission.json'
             },
             'tags': {
                 'storage': liststorage.StringListStorage,
@@ -121,8 +124,15 @@ def __init__(self, request=None, response=None):
     def get(self, cont_name, list_name, **kwargs):
         _id = kwargs.pop('cid')
         permchecker, storage, _, _, keycheck = self._initialize_request(cont_name, list_name, _id, query_params=kwargs)
+        project_storage = ProjectStorage()
         try:
-            result = keycheck(permchecker(storage.exec_op))('GET', _id, query_params=kwargs)
+            # Check to see if list_name in phi lists of site level or project
+            if list_name in ["tags", "notes"]:
+                phi = list_name in (project_storage.get_phi_fields("site")["fields"] + project_storage.get_phi_fields(get_project_id(cont_name,_id))["fields"]) and check_phi_enabled(cont_name, _id)
+                log.debug("Phi: {}".format(phi))
+                result = keycheck(permchecker(storage.exec_op))('GET', _id, query_params=kwargs, phi=phi)
+            else:
+                result = keycheck(permchecker(storage.exec_op))('GET', _id, query_params=kwargs)
         except APIStorageException as e:
             self.abort(400, e.message)
 
@@ -188,7 +198,7 @@ def _initialize_request(self, cont_name, list_name, _id, query_params=None):
             query_params = None
         container = storage.get_container(_id, query_params)
         if container is not None:
-            if self.superuser_request or self.user_is_admin:
+            if self.superuser_request:
                 permchecker = always_ok
             elif self.public_request:
                 permchecker = listauth.public_request(self, container)
@@ -213,6 +223,8 @@ def post(self, cont_name, list_name, **kwargs):
         _id = kwargs.get('cid')
         result = super(PermissionsListHandler, self).post(cont_name, list_name, **kwargs)
         payload = self.request.json_body
+        if cont_name == "groups" and not payload.get("phi-access"):
+            payload["phi-access"] = True
 
         if cont_name == 'groups' and self.request.params.get('propagate') =='true':
             self._propagate_permissions(cont_name, _id, query={'permissions._id' : payload['_id']}, update={'$set': {'permissions.$.access': payload['access'], 'permissions.$.phi-access': payload['phi-access']}})
@@ -319,7 +331,6 @@ class TagsListHandler(ListHandler):
     TagsListHandler overrides put, delete methods of ListHandler to propagate changes to group tags
     If a tag is renamed or deleted at the group level, project, session and acquisition tags will also be renamed/deleted
     """
-
     def put(self, cont_name, list_name, **kwargs):
         _id = kwargs.get('cid')
         result = super(TagsListHandler, self).put(cont_name, list_name, **kwargs)