Skip to content

Commit

Permalink
Merge pull request #875 from uc-cdis/feat/visa_sync_loging
Browse files Browse the repository at this point in the history 
(PXP-7717): Implement ad-hoc usersync for single user after logging in through RAS
  • Loading branch information
@BinamB
BinamB committed Feb 23, 2021
2 parents 1bb9084 + 1b4379f commit 3fef376
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 99 deletions.
25 changes: 25 additions & 0 deletions fence/blueprints/login/ras.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import flask
import jwt
import os
from flask_sqlalchemy_session import current_session

from fence.models import GA4GHVisaV1, IdentityProvider

from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback

from fence.config import config
from fence.scripting.fence_create import init_syncer


class RASLogin(DefaultOAuth2Login):
Expand Down Expand Up @@ -69,3 +71,26 @@ def post_login(self, user, token_result):
flask.current_app.ras_client.store_refresh_token(
user=user, refresh_token=refresh_token, expires=expires
)

# Check if user has any project_access from a previous session or from usersync
# if not do an on-the-fly usersync for this user to give them instant access after logging in through RAS
if not user.project_access:
# Close previous db sessions. Leaving it open causes a race condition where we're viewing user.project_access while trying to update it in usersync
# not closing leads to partially updated records
current_session.close()
DB = os.environ.get("FENCE_DB") or config.get("DB")
if DB is None:
try:
from fence.settings import DB
except ImportError:
pass
dbGaP = os.environ.get("dbGaP") or config.get("dbGaP")
if not isinstance(dbGaP, list):
dbGaP = [dbGaP]

sync = init_syncer(
dbGaP,
None,
DB,
)
sync.sync_single_user_visas(user, current_session)
282 changes: 183 additions & 99 deletions fence/sync/sync_users.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,32 +565,15 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True):
"tags": tags,
}

if dbgap_project not in self.project_mapping:
self._add_dbgap_project_for_user(
dbgap_project,
privileges,
username,
sess,
user_projects,
dbgap_config,
)
for element_dict in self.project_mapping.get(dbgap_project, []):
try:
phsid_privileges = {
element_dict["auth_id"]: set(privileges)
}

# need to add dbgap project to arborist
if self.arborist_client:
self._add_dbgap_study_to_arborist(
element_dict["auth_id"], dbgap_config
)
self._process_dbgap_project(
dbgap_project,
privileges,
username,
sess,
user_projects,
dbgap_config,
)

if username not in user_projects:
user_projects[username] = {}
user_projects[username].update(phsid_privileges)
except ValueError as e:
self.logger.info(e)
return user_projects, user_info

def _add_dbgap_project_for_user(
Expand Down Expand Up @@ -703,7 +686,9 @@ def sync_two_phsids_dict(
elif source2:
self.auth_source[user].add(source2)

def sync_to_db_and_storage_backend(self, user_project, user_info, sess):
def sync_to_db_and_storage_backend(
self, user_project, user_info, sess, single_visa_sync=False
):
"""
sync user access control to database and storage backend
Expand Down Expand Up @@ -762,10 +747,11 @@ def sync_to_db_and_storage_backend(self, user_project, user_info, sess):
# pass the original, non-lowered user_info dict
self._upsert_userinfo(sess, user_info)

self._revoke_from_storage(
to_delete, sess, google_bulk_mapping=google_bulk_mapping
)
self._revoke_from_db(sess, to_delete)
if not single_visa_sync:
self._revoke_from_storage(
to_delete, sess, google_bulk_mapping=google_bulk_mapping
)
self._revoke_from_db(sess, to_delete)

self._grant_from_storage(
to_add,
Expand All @@ -791,7 +777,8 @@ def sync_to_db_and_storage_backend(self, user_project, user_info, sess):
)
self._update_from_db(sess, to_update, user_project_lowercase)

self._validate_and_update_user_admin(sess, user_info_lowercase)
if not single_visa_sync:
self._validate_and_update_user_admin(sess, user_info_lowercase)

if config["GOOGLE_BULK_UPDATES"]:
self.logger.info("Doing bulk Google update...")
Expand Down Expand Up @@ -1171,6 +1158,95 @@ def parse_projects(self, user_projects):
"""
return {key.lower(): value for key, value in user_projects.items()}

def _process_dbgap_project(
self, dbgap_project, privileges, username, sess, user_projects, dbgap_config
):
if dbgap_project not in self.project_mapping:
self._add_dbgap_project_for_user(
dbgap_project,
privileges,
username,
sess,
user_projects,
dbgap_config,
)

for element_dict in self.project_mapping.get(dbgap_project, []):
try:
phsid_privileges = {element_dict["auth_id"]: set(privileges)}

# need to add dbgap project to arborist
if self.arborist_client:
self._add_dbgap_study_to_arborist(
element_dict["auth_id"], dbgap_config
)

if username not in user_projects:
user_projects[username] = {}
user_projects[username].update(phsid_privileges)

except ValueError as e:
self.logger.info(e)

def _process_user_projects(
self,
user_projects,
enable_common_exchange_area_access,
study_common_exchange_areas,
dbgap_config,
sess,
):
for username in user_projects.keys():
for project in user_projects[username].keys():
phsid = project.split(".")
dbgap_project = phsid[0]
privileges = user_projects[username][project]
if len(phsid) > 1 and self.parse_consent_code:
consent_code = phsid[-1]

# c999 indicates full access to all consents and access
# to a study-specific exchange area
# access to at least one study-specific exchange area implies access
# to the parent study's common exchange area
#
# NOTE: Handling giving access to all consents is done at
# a later time, when we have full information about possible
# consents
self.logger.debug(
f"got consent code {consent_code} from dbGaP project "
f"{dbgap_project}"
)
if (
consent_code == "c999"
and enable_common_exchange_area_access
and dbgap_project in study_common_exchange_areas
):
self.logger.info(
"found study with consent c999 and Fence "
"is configured to parse exchange area data. Giving user "
f"{username} {privileges} privileges in project: "
f"{study_common_exchange_areas[dbgap_project]}."
)
self._add_dbgap_project_for_user(
study_common_exchange_areas[dbgap_project],
privileges,
username,
sess,
user_projects,
dbgap_config,
)

dbgap_project += "." + consent_code

self._process_dbgap_project(
dbgap_project,
privileges,
username,
sess,
user_projects,
dbgap_config,
)

def sync(self):
if self.session:
self._sync(self.session)
Expand Down Expand Up @@ -1917,74 +1993,13 @@ def _sync_visas(self, sess):
user_yaml.projects, user_projects, source1="user_yaml", source2="visa"
)

for username in user_projects.keys():
for project in user_projects[username].keys():
phsid = project.split(".")
dbgap_project = phsid[0]
privileges = user_projects[username][project]
if len(phsid) > 1 and self.parse_consent_code:
consent_code = phsid[-1]

# c999 indicates full access to all consents and access
# to a study-specific exchange area
# access to at least one study-specific exchange area implies access
# to the parent study's common exchange area
#
# NOTE: Handling giving access to all consents is done at
# a later time, when we have full information about possible
# consents
self.logger.debug(
f"got consent code {consent_code} from dbGaP project "
f"{dbgap_project}"
)
if (
consent_code == "c999"
and enable_common_exchange_area_access
and dbgap_project in study_common_exchange_areas
):
self.logger.info(
"found study with consent c999 and Fence "
"is configured to parse exchange area data. Giving user "
f"{username} {privileges} privileges in project: "
f"{study_common_exchange_areas[dbgap_project]}."
)
self._add_dbgap_project_for_user(
study_common_exchange_areas[dbgap_project],
privileges,
username,
sess,
user_projects,
dbgap_config,
)

dbgap_project += "." + consent_code

if dbgap_project not in self.project_mapping:
self._add_dbgap_project_for_user(
dbgap_project,
privileges,
username,
sess,
user_projects,
dbgap_config,
)

for element_dict in self.project_mapping.get(dbgap_project, []):
try:
phsid_privileges = {element_dict["auth_id"]: set(privileges)}

# need to add dbgap project to arborist
if self.arborist_client:
self._add_dbgap_study_to_arborist(
element_dict["auth_id"], dbgap_config
)

if username not in user_projects:
user_projects[username] = {}
user_projects[username].update(phsid_privileges)

except ValueError as e:
self.logger.info(e)
self._process_user_projects(
user_projects,
enable_common_exchange_area_access,
study_common_exchange_areas,
dbgap_config,
sess,
)

# Note: if there are multiple dbgap sftp servers configured
# this parameter is always from the config for the first dbgap sftp server
Expand Down Expand Up @@ -2041,3 +2056,72 @@ def sync_visas(self):
with self.driver.session as s:
self._sync_visas(s)
# if returns with some failure use telemetry file

def sync_single_user_visas(self, user, sess=None):
"""
Sync a single user's visa during login
"""

self.ras_sync_client = RASVisa(logger=self.logger)
dbgap_config = self.dbGaP[0]
enable_common_exchange_area_access = dbgap_config.get(
"enable_common_exchange_area_access", False
)
study_common_exchange_areas = dbgap_config.get(
"study_common_exchange_areas", {}
)

try:
user_yaml = UserYAML.from_file(
self.sync_from_local_yaml_file, encrypted=False, logger=self.logger
)
except (EnvironmentError, AssertionError) as e:
self.logger.error(str(e))
self.logger.error("aborting early")
return

user_projects = dict()
user_info = dict()
projects = {}
info = {}

for visa in user.ga4gh_visas_v1:
project = {}
visa_type = self._pick_sync_type(visa)
encoded_visa = visa.ga4gh_visa
project, info = visa_type._parse_single_visa(
user,
encoded_visa,
visa.expires,
self.parse_consent_code,
sess,
)
projects = {**projects, **project}
user_projects[user.username] = projects
user_info[user.username] = info

user_projects = self.parse_projects(user_projects)

if self.parse_consent_code and enable_common_exchange_area_access:
self.logger.info(
f"using study to common exchange area mapping: {study_common_exchange_areas}"
)

self._process_user_projects(
user_projects,
enable_common_exchange_area_access,
study_common_exchange_areas,
dbgap_config,
sess,
)

if self.parse_consent_code:
self._grant_all_consents_to_c999_users(
user_projects, user_yaml.project_to_resource
)
# update fence db
if user_projects:
self.logger.info("Sync to db and storage backend")
self.sync_to_db_and_storage_backend(user_projects, user_info, sess, True)
else:
self.logger.info("No users for syncing")
23 changes: 23 additions & 0 deletions tests/dbgap_sync/test_user_sync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -725,3 +725,26 @@ def test_user_sync_with_visas(
"phs000298": ["read", "read-storage"],
},
)


@pytest.mark.parametrize("syncer", ["google"], indirect=True)
def test_sync_in_login(
syncer,
db_session,
storage_client,
rsa_private_key,
kid,
monkeypatch,
):
user = models.query_for_user(
session=db_session, username="TESTUSERB"
) # contains no information
assert len(user.project_access) == 0
db_session.close()
syncer.sync_single_user_visas(user, db_session)
user = models.query_for_user(
session=db_session, username="TESTUSERB"
) # contains only visa information
user1 = models.query_for_user(session=db_session, username="USER_1")
assert len(user1.project_access) == 0 # other users are not affected
assert len(user.project_access) == 6

0 comments on commit 3fef376

Please sign in to comment.