Add Site-Level Gear Rules

api/api.py

@@ -171,6 +171,12 @@ def prefix(path, routes):
         ]),
 
 
+        # Site
+
+        route('/<cid:site>/rules',              RulesHandler,          m=['GET', 'POST']),
+        route('/<cid:site>/rules/<rid:{cid}>',  RuleHandler,           m=['GET', 'PUT', 'DELETE']),
+
+
         # Groups
 
         route('/groups',             GroupHandler, h='get_all', m=['GET']),

api/dao/base.py

@@ -4,7 +4,6 @@
 from . import APIStorageException, APIConflictException, APINotFoundException
 from . import consistencychecker
 from . import containerutil
-from . import hierarchy
 from .. import config
 from .. import util
 
@@ -89,7 +88,7 @@ def get_children(self, _id, projection=None, uid=None):
             query = {containerutil.singularize(self.cont_name): _id}
         else:
             query = {containerutil.singularize(self.cont_name): bson.ObjectId(_id)}
-        
+
         if uid:
             query['permissions'] = {'$elemMatch': {'_id': uid}}
         if not projection:
@@ -160,7 +159,7 @@ def update_el(self, _id, payload, unset_payload=None, recursive=False, r_payload
             except bson.InvalidId as e:
                 raise APIStorageException(e.message)
         if recursive and r_payload is not None:
-            hierarchy.propagate_changes(self.cont_name, _id, {}, {'$set': util.mongo_dict(r_payload)})
+            containerutil.propagate_changes(self.cont_name, _id, {}, {'$set': util.mongo_dict(r_payload)})
         return self.dbc.update_one({'_id': _id}, update)
 
     def delete_el(self, _id):

api/dao/containerstorage.py

@@ -8,6 +8,7 @@
 from .. import config
 from ..jobs.gears import validate_gear_config, get_gear
 from ..jobs.jobs import Job
+from ..jobs.rules import copy_site_rules_for_project
 from .base import ContainerStorage
 
 
@@ -43,6 +44,11 @@ class ProjectStorage(ContainerStorage):
     def __init__(self):
         super(ProjectStorage,self).__init__('projects', use_object_id=True)
 
+    def create_el(self, payload):
+        result = super(ProjectStorage, self).create_el(payload)
+        copy_site_rules_for_project(result.inserted_id)
+        return result
+
     def update_el(self, _id, payload, unset_payload=None, recursive=False, r_payload=None, replace_metadata=False):
         result = super(ProjectStorage, self).update_el(_id, payload, unset_payload=unset_payload, recursive=recursive, r_payload=r_payload, replace_metadata=replace_metadata)
 

api/dao/containerutil.py

@@ -1,4 +1,5 @@
 import bson.objectid
+import copy
 
 from . import APIPermissionException
 from .. import config
@@ -16,6 +17,48 @@
 }
 PLURAL_TO_SINGULAR = {p: s for s, p in SINGULAR_TO_PLURAL.iteritems()}
 
+def propagate_changes(cont_name, _id, query, update):
+    """
+    Propagates changes down the heirarchy tree.
+
+    cont_name and _id refer to top level container (which will not be modified here)
+    """
+
+
+    if cont_name == 'groups':
+        project_ids = [p['_id'] for p in config.db.projects.find({'group': _id}, [])]
+        session_ids = [s['_id'] for s in config.db.sessions.find({'project': {'$in': project_ids}}, [])]
+
+        project_q = copy.deepcopy(query)
+        project_q['_id'] = {'$in': project_ids}
+        session_q = copy.deepcopy(query)
+        session_q['_id'] = {'$in': session_ids}
+        acquisition_q = copy.deepcopy(query)
+        acquisition_q['session'] = {'$in': session_ids}
+
+        config.db.projects.update_many(project_q, update)
+        config.db.sessions.update_many(session_q, update)
+        config.db.acquisitions.update_many(acquisition_q, update)
+
+
+    # Apply change to projects
+    elif cont_name == 'projects':
+        session_ids = [s['_id'] for s in config.db.sessions.find({'project': _id}, [])]
+
+        session_q = copy.deepcopy(query)
+        session_q['project'] = _id
+        acquisition_q = copy.deepcopy(query)
+        acquisition_q['session'] = {'$in': session_ids}
+
+        config.db.sessions.update_many(session_q, update)
+        config.db.acquisitions.update_many(acquisition_q, update)
+
+    elif cont_name == 'sessions':
+        query['session'] = _id
+        config.db.acquisitions.update_many(query, update)
+    else:
+        raise ValueError('changes can only be propagated from group, project or session level')
+
 
 def add_id_to_subject(subject, pid):
     """

api/dao/hierarchy.py

@@ -9,6 +9,7 @@
 from .. import files
 from .. import util
 from .. import config
+from .base import ContainerStorage
 from ..auth import has_access
 from . import APIStorageException, APINotFoundException, APIPermissionException, containerutil
 
@@ -155,7 +156,7 @@ def propagate_changes(cont_name, _id, query, update):
         config.db.sessions.update_many(session_q, update)
         config.db.acquisitions.update_many(acquisition_q, update)
 
-    
+
     # Apply change to projects
     elif cont_name == 'projects':
         session_ids = [s['_id'] for s in config.db.sessions.find({'project': _id}, [])]
@@ -356,7 +357,7 @@ def _find_or_create_destination_project(group_id, project_label, timestamp, user
                 'created': timestamp,
                 'modified': timestamp
         }
-        result = config.db.projects.insert_one(project)
+        result = ContainerStorage.factory('project').create_el(project)
         project['_id'] = result.inserted_id
     return project
 

api/handlers/listhandler.py

@@ -16,7 +16,7 @@
 from ..dao import noop
 from ..dao import liststorage
 from ..dao import APIStorageException
-from ..dao import hierarchy
+from ..dao import containerutil
 from ..web.request import log_access, AccessType
 
 
@@ -251,7 +251,7 @@ def _propagate_permissions(self, cont_name, _id, query=None, update=None):
             query = {}
         if cont_name == 'groups':
             try:
-                hierarchy.propagate_changes(cont_name, _id, query, update)
+                containerutil.propagate_changes(cont_name, _id, query, update)
             except APIStorageException as e:
                 self.abort(400, e.message)
         elif cont_name == 'projects':
@@ -260,7 +260,7 @@ def _propagate_permissions(self, cont_name, _id, query=None, update=None):
                 update = {'$set': {
                     'permissions': config.db[cont_name].find_one({'_id': oid},{'permissions': 1})['permissions']
                 }}
-                hierarchy.propagate_changes(cont_name, oid, {}, update)
+                containerutil.propagate_changes(cont_name, oid, {}, update)
             except APIStorageException:
                 self.abort(500, 'permissions not propagated from {} {} down hierarchy'.format(cont_name, _id))
 
@@ -339,7 +339,7 @@ def _propagate_group_tags(self, cont_name, _id, query, update):
         method to propagate tag changes from a group to its projects, sessions and acquisitions
         """
         try:
-            hierarchy.propagate_changes(cont_name, _id, query, update)
+            containerutil.propagate_changes(cont_name, _id, query, update)
         except APIStorageException:
             self.abort(500, 'tag change not propagated from group {}'.format(_id))
 

api/jobs/gears.py

@@ -12,7 +12,7 @@
 
 from .. import config
 from .jobs import Job
-from ..dao import APIValidationException
+from ..dao import APIValidationException, APINotFoundException
 from ..dao.base import ContainerStorage
 
 log = config.log
@@ -46,7 +46,7 @@ def get_gear_by_name(name):
     gear_doc = list(config.db.gears.find({'gear.name': name}).sort('created', pymongo.DESCENDING))
 
     if len(gear_doc) == 0 :
-        raise Exception('Unknown gear ' + name)
+        raise APINotFoundException('Unknown gear ' + name)
 
     return gear_doc[0]
 

api/jobs/handlers.py

@@ -10,16 +10,16 @@
 from .. import upload
 from .. import util
 from ..auth import require_login, has_access
-from ..dao import APIPermissionException
-from ..dao.containerstorage import AcquisitionStorage
+from ..dao import APIPermissionException, APINotFoundException
+from ..dao.containerstorage import ProjectStorage, AcquisitionStorage
 from ..dao.containerutil import create_filereference_from_dictionary, create_containerreference_from_dictionary, create_containerreference_from_filereference, ContainerReference
 from ..web import base
 from ..validators import InputValidationException
 from .. import config
 from . import batch
-from ..validators import validate_data
+from ..validators import validate_data, verify_payload_exists
 
-from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container
+from .gears import validate_gear_config, get_gears, get_gear, get_invocation_schema, remove_gear, upsert_gear, suggest_container, get_gear_by_name
 from .jobs import Job, Logs
 from .batch import check_state, update
 from .queue import Queue
@@ -136,92 +136,117 @@ class RulesHandler(base.RequestHandler):
     def get(self, cid):
         """List rules"""
 
-        project = config.db.projects.find_one({'_id': bson.ObjectId(cid)})
+        projection = None
 
-        if project and (self.superuser_request or has_access(self.uid, project, 'ro')):
-            return config.db.project_rules.find({'project_id' : cid})
+        if cid == 'site':
+            if self.public_request:
+                raise APIPermissionException('Viewing site-level rules requires login.')
+            projection = {'project_id': 0}
         else:
-            self.abort(404, 'Project not found')
+            project = ProjectStorage().get_container(cid, projection={'permissions': 1})
+            if not self.user_is_admin and not has_access(self.uid, project, 'ro'):
+                raise APIPermissionException('User does not have access to project {} rules'.format(cid))
 
+        return config.db.project_rules.find({'project_id' : cid}, projection=projection)
+
+
+    @verify_payload_exists
     def post(self, cid):
         """Add a rule"""
 
-        project = config.db.projects.find_one({'_id': bson.ObjectId(cid)})
-
-        if project:
-            if self.superuser_request or has_access(self.uid, project, 'admin'):
-                doc = self.request.json
+        if cid == 'site':
+            if not self.user_is_admin:
+                raise APIPermissionException('Adding site-level rules can only be done by a site admin.')
+        else:
+            project = ProjectStorage().get_container(cid, projection={'permissions': 1})
+            if not self.user_is_admin and not has_access(self.uid, project, 'admin'):
+                raise APIPermissionException('Adding rules to a project can only be done by a project admin.')
 
-                validate_data(doc, 'rule-add.json', 'input', 'POST', optional=True)
+        doc = self.request.json
 
-                doc['project_id'] = cid
+        validate_data(doc, 'rule-add.json', 'input', 'POST', optional=True)
+        try:
+            get_gear_by_name(doc['alg'])
+        except APINotFoundException:
+            self.abort(400, 'Cannot find gear for alg {}, alg not valid'.format(doc['alg']))
 
-                result = config.db.project_rules.insert_one(doc)
-                return { '_id': result.inserted_id }
+        doc['project_id'] = cid
 
-            elif has_access(self.uid, project, 'ro'):
-                self.abort(403, 'Adding rules to a project can only be done by a project admin.')
+        result = config.db.project_rules.insert_one(doc)
+        return { '_id': result.inserted_id }
 
-        self.abort(404, 'Project not found')
 
 class RuleHandler(base.RequestHandler):
 
     def get(self, cid, rid):
         """Get rule"""
 
-        project = config.db.projects.find_one({'_id': bson.ObjectId(cid)})
-
-        if project and (self.superuser_request or has_access(self.uid, project, 'ro')):
-            result = config.db.project_rules.find_one({'project_id' : cid, '_id': bson.ObjectId(rid)})
-            if result:
-                return result
-            else:
-                self.abort(404, 'Rule not found')
+        projection = None
+        if cid == 'site':
+            if self.public_request:
+                raise APIPermissionException('Viewing site-level rules requires login.')
+            projection = {'project_id': 0}
         else:
-            self.abort(404, 'Project not found')
+            project = ProjectStorage().get_container(cid, projection={'permissions': 1})
+            if not self.user_is_admin and not has_access(self.uid, project, 'ro'):
+                raise APIPermissionException('User does not have access to project {} rules'.format(cid))
+
+        result = config.db.project_rules.find_one({'project_id' : cid, '_id': bson.ObjectId(rid)}, projection=projection)
 
+        if not result:
+            raise APINotFoundException('Rule not found.')
+
+        return result
+
+
+    @verify_payload_exists
     def put(self, cid, rid):
         """Change a rule"""
 
-        project = config.db.projects.find_one({'_id': bson.ObjectId(cid)})
+        if cid == 'site':
+            if not self.user_is_admin:
+                raise APIPermissionException('Modifying site-level rules can only be done by a site admin.')
+        else:
+            project = ProjectStorage().get_container(cid, projection={'permissions': 1})
+            if not self.user_is_admin and not has_access(self.uid, project, 'admin'):
+                raise APIPermissionException('Modifying project rules can only be done by a project admin.')
 
-        if project:
-            if self.superuser_request or has_access(self.uid, project, 'admin'):
-                result = config.db.project_rules.find_one({'project_id' : cid, '_id': bson.ObjectId(rid)})
-                if result is None:
-                    self.abort(404, 'Rule not found')
+        doc = config.db.project_rules.find_one({'project_id' : cid, '_id': bson.ObjectId(rid)})
 
-                doc = self.request.json
-                validate_data(doc, 'rule-update.json', 'input', 'POST', optional=True)
+        if not doc:
+            raise APINotFoundException('Rule not found.')
 
-                doc['_id']        = result['_id']
-                doc['project_id'] = cid
+        updates = self.request.json
+        validate_data(updates, 'rule-update.json', 'input', 'POST', optional=True)
+        if updates.get('alg'):
+            try:
+                get_gear_by_name(updates['alg'])
+            except APINotFoundException:
+                self.abort(400, 'Cannot find gear for alg {}, alg not valid'.format(updates['alg']))
 
-                config.db.project_rules.update_one({'_id': bson.ObjectId(rid)}, {'$set': doc })
-                return
+        doc.update(updates)
+        config.db.project_rules.replace_one({'_id': bson.ObjectId(rid)}, doc)
 
-            elif has_access(self.uid, project, 'ro'):
-                self.abort(403, 'Changing project rules can only be done by a project admin.')
+        return
 
-        self.abort(404, 'Project not found')
 
     def delete(self, cid, rid):
         """Remove a rule"""
 
-        project = config.db.projects.find_one({'_id': bson.ObjectId(cid)})
-
-        if project:
-            if self.superuser_request or has_access(self.uid, project, 'admin'):
+        if cid == 'site':
+            if not self.user_is_admin:
+                raise APIPermissionException('Modifying site-level rules can only be done by a site admin.')
+        else:
+            project = ProjectStorage().get_container(cid, projection={'permissions': 1})
+            if not self.user_is_admin and not has_access(self.uid, project, 'admin'):
+                raise APIPermissionException('Modifying project rules can only be done by a project admin.')
 
-                result = config.db.project_rules.delete_one({'project_id' : cid, '_id': bson.ObjectId(rid)})
-                if result.deleted_count != 1:
-                    self.abort(404, 'Rule not found')
-                return
 
-            elif has_access(self.uid, project, 'ro'):
-                self.abort(403, 'Changing project rules can only be done by a project admin.')
+        result = config.db.project_rules.delete_one({'project_id' : cid, '_id': bson.ObjectId(rid)})
+        if result.deleted_count != 1:
+            raise APINotFoundException('Rule not found.')
+        return
 
-        self.abort(404, 'Project not found')
 
 class JobsHandler(base.RequestHandler):
     """Provide /jobs API routes."""