Skip to content

Commit

Permalink
Merge pull request #928 from uc-cdis/test/parse_visas
Browse files Browse the repository at this point in the history 
(PXP-7717) Implement ad-hoc usersync for single user after logging in through RAS
  • Loading branch information
@BinamB
BinamB committed Jun 23, 2021
2 parents 00cff2c + e9b1c3d commit 4cfe00b
Show file tree
Hide file tree
Showing 5 changed files with 711 additions and 466 deletions.
25 changes: 22 additions & 3 deletions fence/blueprints/login/ras.py
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
import flask
import jwt
import os
from distutils.util import strtobool
from authutils.errors import JWTError
from authutils.token.core import validate_jwt
from authutils.token.keys import get_public_key_for_token
from cdislogging import get_logger
from flask_sqlalchemy_session import current_session
import urllib.request, urllib.error
from urllib.parse import urlparse, parse_qs

from fence.models import GA4GHVisaV1, IdentityProvider
from gen3authz.client.arborist.client import ArboristClient

from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback

from fence.config import config
from fence.scripting.fence_create import init_syncer
from fence.utils import get_valid_expiration
Expand Down Expand Up @@ -133,20 +133,38 @@ def post_login(self, user=None, token_result=None):
user=user, refresh_token=refresh_token, expires=expires + issued_time
)

global_parse_visas_on_login = config["GLOBAL_PARSE_VISAS_ON_LOGIN"]
usersync = config.get("USERSYNC", {})
sync_from_visas = usersync.get("sync_from_visas", False)
parse_visas = global_parse_visas_on_login or (
global_parse_visas_on_login == None
and (
strtobool(query_params.get("parse_visas")[0])
if query_params.get("parse_visas")
else False
)
)
# if sync_from_visas and (global_parse_visas_on_login or global_parse_visas_on_login == None):
# Check if user has any project_access from a previous session or from usersync AND if fence is configured to use visas as authZ source
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
if not user.project_access and sync_from_visas:
# If GLOBAL_PARSE_VISAS_ON_LOGIN is true then we want to run it regardless of whether or not the client sent parse_visas on request
if sync_from_visas and parse_visas and not user.project_access:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()

DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass

arborist = ArboristClient(
arborist_base_url=config["ARBORIST"],
logger=get_logger("user_syncer.arborist_client"),
authz_provider="user-sync",
)
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]
Expand All @@ -155,6 +173,7 @@ def post_login(self, user=None, token_result=None):
dbGaP,
None,
DB,
arborist=arborist,
)
sync.sync_single_user_visas(user, current_session)

Expand Down
6 changes: 6 additions & 0 deletions fence/config-default.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,12 @@ GA4GH_VISA_ISSUER_ALLOWLIST:
- 'https://stsstg.nih.gov'
# Number of projects that can be registered to a Google Service Accont
SERVICE_ACCOUNT_LIMIT: 6

# Global sync visas during login
# None(Default): Allow per client i.e. a fence client can pick whether or not to sync their visas during login with parse_visas param in /authorization endpoint
# True: Parse for all clients i.e. a fence client will always sync their visas during login
# False: Parse for no clients i.e. a fence client will not be able to sync visas during login even with parse_visas param
GLOBAL_PARSE_VISAS_ON_LOGIN:
# Settings for usersync with visas
USERSYNC:
sync_from_visas: false
Expand Down
138 changes: 98 additions & 40 deletions fence/sync/sync_users.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import subprocess as sp
import yaml
import copy

from contextlib import contextmanager
from collections import defaultdict
from csv import DictReader
Expand Down Expand Up @@ -1196,6 +1197,7 @@ def _process_user_projects(
dbgap_config,
sess,
):

for username in user_projects.keys():
for project in user_projects[username].keys():
phsid = project.split(".")
Expand Down Expand Up @@ -1392,6 +1394,8 @@ def _sync(self, sess):
"Could not synchronize authorization info successfully to arborist"
)
exit(1)
else:
self.logger.error("No arborist client set; skipping arborist sync")

# Logging authz source
for u, s in self.auth_source.items():
Expand Down Expand Up @@ -1584,7 +1588,9 @@ def _update_arborist(self, session, user_yaml):

return True

def _update_authz_in_arborist(self, session, user_projects, user_yaml=None):
def _update_authz_in_arborist(
self, session, user_projects, user_yaml=None, single_user_sync=False
):
"""
Assign users policies in arborist from the information in
``user_projects`` and optionally a ``user_yaml``.
Expand Down Expand Up @@ -1621,26 +1627,31 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None):
# get list of users from arborist to make sure users that are completely removed
# from authorization sources get policies revoked
arborist_user_projects = {}
try:
arborist_users = self.arborist_client.get_users().json["users"]
# construct user information, NOTE the lowering of the username. when adding/
# removing access, the case in the Fence db is used. For combining access, it is
# case-insensitive, so we lower
arborist_user_projects = {
user["name"].lower(): {} for user in arborist_users
}
except (ArboristError, KeyError, AttributeError) as error:
# TODO usersync should probably exit with non-zero exit code at the end,
# but sync should continue from this point so there are no partial
# updates
self.logger.warning(
"Could not get list of users in Arborist, continuing anyway. "
"WARNING: this sync will NOT remove access for users no longer in "
f"authorization sources. Error: {error}"
)
if not single_user_sync:
try:
arborist_users = self.arborist_client.get_users().json["users"]

# construct user information, NOTE the lowering of the username. when adding/
# removing access, the case in the Fence db is used. For combining access, it is
# case-insensitive, so we lower
arborist_user_projects = {
user["name"].lower(): {} for user in arborist_users
}
except (ArboristError, KeyError, AttributeError) as error:
# TODO usersync should probably exit with non-zero exit code at the end,
# but sync should continue from this point so there are no partial
# updates
self.logger.warning(
"Could not get list of users in Arborist, continuing anyway. "
"WARNING: this sync will NOT remove access for users no longer in "
f"authorization sources. Error: {error}"
)

# update the project info with users from arborist
self.sync_two_phsids_dict(arborist_user_projects, user_projects)

# update the project info with users from arborist
self.sync_two_phsids_dict(arborist_user_projects, user_projects)
policy_id_list = []
policies = []

for username, user_project_info in user_projects.items():
self.logger.info("processing user `{}`".format(username))
Expand All @@ -1650,7 +1661,6 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None):

self.arborist_client.create_user_if_not_exist(username)
self.arborist_client.revoke_all_policies_for_user(username)

for project, permissions in user_project_info.items():

# check if this is a dbgap project, if it is, we need to get the right
Expand All @@ -1668,7 +1678,6 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None):
"resource paths for project {}: {}".format(project, paths)
)
self.logger.debug("permissions: {}".format(permissions))

for permission in permissions:
# "permission" in the dbgap sense, not the arborist sense
if permission not in self._created_roles:
Expand All @@ -1692,24 +1701,50 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None):
# format project '/x/y/z' -> 'x.y.z'
# so the policy id will be something like 'x.y.z-create'
policy_id = _format_policy_id(path, permission)
if policy_id not in self._created_policies:
try:
self.arborist_client.update_policy(
policy_id,
{
"description": "policy created by fence sync",
"role_ids": [permission],
"resource_paths": [path],
},
create_if_not_exist=True,
)
except ArboristError as e:
self.logger.info(
"not creating policy in arborist; {}".format(str(e))
)
self._created_policies.add(policy_id)

self.arborist_client.grant_user_policy(username, policy_id)
if not single_user_sync:
if policy_id not in self._created_policies:
try:
self.arborist_client.update_policy(
policy_id,
{
"description": "policy created by fence sync",
"role_ids": [permission],
"resource_paths": [path],
},
create_if_not_exist=True,
)
except ArboristError as e:
self.logger.info(
"not creating policy in arborist; {}".format(
str(e)
)
)
self._created_policies.add(policy_id)
self.arborist_client.grant_user_policy(username, policy_id)

if single_user_sync:
policy_id_list.append(policy_id)
policy_json = {
"id": policy_id,
"description": "policy created by fence sync",
"role_ids": [permission],
"resource_paths": [path],
}
policies.append(policy_json)

if single_user_sync:
try:
self.arborist_client.update_bulk_policy(policies)
self.arborist_client.grant_bulk_user_policy(
username, policy_id_list
)
except Exception as e:
self.logger.info(
"Couldn't update bulk policy for user {}: {}".format(
username, e
)
)

if user_yaml:
for policy in user_yaml.policies.get(username, []):
Expand Down Expand Up @@ -2052,6 +2087,9 @@ def _sync_visas(self, sess):
"Could not synchronize authorization info successfully to arborist"
)
exit(1)
else:
self.logger.error("No arborist client set; skipping arborist sync")

# Logging authz source
for u, s in self.auth_source.items():
self.logger.info("Access for user {} from {}".format(u, s))
Expand Down Expand Up @@ -2126,9 +2164,29 @@ def sync_single_user_visas(self, user, sess=None):
self._grant_all_consents_to_c999_users(
user_projects, user_yaml.project_to_resource
)

# update fence db
if user_projects:
self.logger.info("Sync to db and storage backend")
self.sync_to_db_and_storage_backend(user_projects, user_info, sess, True)
self.sync_to_db_and_storage_backend(
user_projects, user_info, sess, single_visa_sync=True
)
else:
self.logger.info("No users for syncing")

# update arborist db (user access)
if self.arborist_client:
self.logger.info("Synchronizing arborist with authorization info...")
success = self._update_authz_in_arborist(
sess, user_projects, user_yaml=user_yaml, single_user_sync=True
)
if success:
self.logger.info(
"Finished synchronizing authorization info to arborist"
)
else:
self.logger.error(
"Could not synchronize authorization info successfully to arborist"
)
else:
self.logger.error("No arborist client set; skipping arborist sync")
Loading

0 comments on commit 4cfe00b

Please sign in to comment.