Grouper integrations for project-level admins

Summary: This diff builds the foundation for project-level admins. This includes a model change to add a new column for `project_permissions` to User. There is a background task that syncs project admins with Grouper, and also create a new user if a user should be an admin or a project admin but has never used Changes before. This does NOT have any user-facing changes.

Test Plan: unit tests + local testing

Reviewers: anupc

Reviewed By: anupc

Subscribers: changesbot, herb, kylec

Differential Revision: https://tails.corp.dropbox.com/D221468

changes/config.py

@@ -89,6 +89,7 @@ def create_app(_read_config=True, **config):
     app.config['REDIS_URL'] = 'redis://localhost/0'
     app.config['GROUPER_API_URL'] = 'https://localhost/'
     app.config['GROUPER_PERMISSIONS_ADMIN'] = 'changes.prod.admin'
+    app.config['GROUPER_PERMISSIONS_PROJECT_ADMIN'] = 'changes.prod.project.admin'
     app.config['GROUPER_EXCLUDED_ROLES'] = ['np-owner']
     app.config['DEBUG'] = True
     app.config['HTTP_PORT'] = 5000

changes/jobs/sync_grouper.py

@@ -3,24 +3,32 @@
 import urlparse
 
 from flask import current_app
-from typing import Iterable, Set  # NOQA
+from typing import Dict, Iterable, Set  # NOQA
 
 from changes.config import db, statsreporter
+from changes.db.utils import create_or_update
 from changes.models.user import User
 
 
 logger = logging.getLogger('grouper.sync')
 
 
+class GrouperApiError(Exception):
+    pass
+
+
 def sync_grouper():
     # type: () -> None
-    """This function is meant as a Celery task. It connects to Grouper, gets
-    all users who should be admin, and makes sure that those users and only
-    those users are admin.
+    """This function is meant as a Celery task. It connects to Grouper, and
+    does two sets of syncs:
+    - global admin
+    - project-level admins
     """
     try:
         admin_emails = _get_admin_emails_from_grouper()
         _sync_admin_users(admin_emails)
+        project_admin_mapping = _get_project_admin_mapping_from_grouper()
+        _sync_project_admin_users(project_admin_mapping)
     except Exception:
         logger.exception("An error occurred during Grouper sync.")
         statsreporter.stats().set_gauge('grouper_sync_error', 1)
@@ -36,10 +44,16 @@ def _get_admin_emails_from_grouper():
 
     Returns:
         set[basestring]: a set of emails of admin users
+    Raises:
+        GrouperApiError - If there is an error on returned from Grouper
     """
     url = urlparse.urljoin(current_app.config['GROUPER_API_URL'],
                            '/permissions/{}'.format(current_app.config['GROUPER_PERMISSIONS_ADMIN']))
-    groups = requests.get(url).json()['data']['groups']
+    response = requests.get(url).json()
+    if 'errors' in response:
+        message = '\n'.join([x['message'] for x in response['errors']])
+        raise GrouperApiError(message)
+    groups = response['data']['groups']
 
     admin_users = set()
     for _, group in groups.iteritems():
@@ -54,7 +68,8 @@ def _sync_admin_users(admin_emails):
     """Take a look at the Changes user database. Every user with email in
     `admin_emails` should become a Changes admin, and every user already
     an admin whose email is not in `admin_emails` will have their
-    admin privileges revoked.
+    admin privileges revoked. Note that if a user who should be an admin
+    does not exist in the Changes database, the user is created.
 
     Args:
         admin_emails (iterable[basestring]): an iterable of usernames of
@@ -69,10 +84,80 @@ def _sync_admin_users(admin_emails):
     }, synchronize_session=False)
 
     # give access for people who should have access
+    for email in admin_emails:
+        create_or_update(User, where={
+            'email': email,
+        }, values={
+            'is_admin': True,
+        })
+    db.session.commit()
+
+
+def _get_project_admin_mapping_from_grouper():
+    # type: () -> Dict[str, Set[str]]
+    """This connects to Grouper and retrieves users with project admin
+    permissions.
+
+    Returns:
+        Dict[str, Set[str]]: The mapping from emails to project patterns
+            (the same ones that goes into User.project_permissions)
+    Raises:
+        GrouperApiError - If there is an error on returned from Grouper
+    """
+    url = urlparse.urljoin(current_app.config['GROUPER_API_URL'],
+                           '/permissions/{}'.format(current_app.config['GROUPER_PERMISSIONS_PROJECT_ADMIN']))
+    response = requests.get(url).json()
+    if 'errors' in response:
+        if len(response['errors']) == 1 and response['errors'][0]['code'] == 404:
+            # this just means that we have not assigned any project admins on
+            # grouper. Unlike with global admin, this is a totally valid
+            # scenario
+            return dict()
+        message = '\n'.join([x['message'] for x in response['errors']])
+        raise GrouperApiError(message)
+    groups = response['data']['groups']
+    mapping = dict()  # type: Dict[str, Set[str]]
+    for _, group in groups.iteritems():
+        pattern_set = set()
+        for p in group['permissions']:
+            if p['permission'] == current_app.config['GROUPER_PERMISSIONS_PROJECT_ADMIN']:
+                # based on my inspection of Grouper API, this condition above is
+                # probably always true, but let's check anyways to be confident
+                pattern = p['argument']
+                if len(pattern) > 0:  # an empty string = unargumented permission
+                    pattern_set.add(pattern)
+        if len(pattern_set) > 0:
+            # this is most likely always true, unless the Grouper API somehow
+            # lists a group that has nothing to do with this permission
+            for email, user in group['users'].iteritems():
+                if user['rolename'] not in current_app.config['GROUPER_EXCLUDED_ROLES']:
+                    existing_pattern_set = mapping.get(email, set())
+                    mapping[email] = existing_pattern_set.union(pattern_set)
+    return mapping
+
+
+def _sync_project_admin_users(project_admin_mapping):
+    # type: (Dict[str, Set[str]]) -> None
+    """This synchronizes the Changes user database so that only people
+    in `project_admin_mapping` are project admins, and that they are
+    admins only for the projects they have permissions to.  Note that
+    if a user who should be a project admin does not exist in the Changes
+    database, the user is created.
+
+    Args:
+        project_admin_mapping (Dict[str, Set[str]]): The mapping from
+            user emails to project patterns
+    """
     User.query.filter(
-        User.email.in_(admin_emails),
-        User.is_admin.is_(False),
+        ~User.email.in_(project_admin_mapping.keys()),
+        ~User.project_permissions.is_(None),
     ).update({
-        'is_admin': True,
+        'project_permissions': None
     }, synchronize_session=False)
+    for email, project_permissions in project_admin_mapping.iteritems():
+        create_or_update(User, where={
+            'email': email,
+        }, values={
+            'project_permissions': list(project_permissions)
+        })
     db.session.commit()

changes/models/user.py

@@ -2,6 +2,7 @@
 
 from datetime import datetime
 from sqlalchemy import Boolean, Column, String, DateTime
+from sqlalchemy.dialects.postgresql import ARRAY
 
 from changes.config import db
 from changes.db.types.guid import GUID
@@ -18,6 +19,13 @@ class User(db.Model):
     is_admin = Column(Boolean, default=False, nullable=False)
     date_created = Column(DateTime, default=datetime.utcnow)
 
+    # this keeps track of the list of wildcard patterns of project names that
+    # the user has access to. For example:
+    # - `foo` matches `foo`
+    # - `foo.*` matches anything starting with `foo.`, like `foo.staging`, `foo.`
+    # - `*foo*` matches anything containing `foo`, like `barfoobar`, `foo`
+    project_permissions = Column(ARRAY(String(256)), nullable=True)
+
     def __init__(self, **kwargs):
         super(User, self).__init__(**kwargs)
         if not self.id: