From 485cc3405c0261411cee422cec9de27a84f04e19 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 18 Feb 2022 10:43:33 -0600 Subject: [PATCH 01/14] allow downloading custom phsids --- fence/config-default.yaml | 2 ++ fence/sync/sync_users.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index c3471ec5ce..6f88265ac9 100755 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -890,6 +890,8 @@ SERVICE_ACCOUNT_LIMIT: 6 GLOBAL_PARSE_VISAS_ON_LOGIN: # Settings for usersync with visas USERSYNC: + # allowed list of custom project ids that can be processed by usersync + allowed_custom_project_ids: ["CUSTOM-*"] sync_from_visas: false # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed fallback_to_dbgap_sftp: false diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 8ec52d28b6..a29d20fcdf 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -340,7 +340,7 @@ def __init__( ) @staticmethod - def _match_pattern(filepath, encrypted=True): + def _match_pattern(filepath, custom_id_patterns, encrypted=True): """ Check if the filename matches dbgap access control file pattern @@ -351,11 +351,15 @@ def _match_pattern(filepath, encrypted=True): Returns: bool: whether the pattern matches """ - pattern = r"authentication_file_phs(\d{6}).(csv|txt)" - if encrypted: - pattern += ".enc" - pattern += "$" - return re.match(pattern, os.path.basename(filepath)) + custom_id_patterns.insert(0, "phs(\d{6})") + for pattern in custom_id_patterns: + pattern = r"authentication_file_{}.(csv|txt)".format(pattern) + if encrypted: + pattern += ".enc" + pattern += "$" + if re.match(pattern, os.path.basename(filepath)): + return True + return False def _get_from_sftp_with_proxy(self, server, path): """ @@ -472,6 +476,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): # parse dbGaP sftp server information dbgap_key = dbgap_config.get("decrypt_key", None) parse_consent_code = dbgap_config.get("parse_consent_code", True) + custom_id_patterns = dbgap_config.get("allowed_custom_ids", []) enable_common_exchange_area_access = dbgap_config.get( "enable_common_exchange_area_access", False ) @@ -488,7 +493,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): if os.stat(filepath).st_size == 0: self.logger.warning("Empty file {}".format(filepath)) continue - if not self._match_pattern(filepath, encrypted=encrypted): + if not self._match_pattern(filepath, custom_id_patterns=custom_id_patterns, encrypted=encrypted): self.logger.warning( "Filename {} does not match dbgap access control filename pattern;" " this could mean that the filename has an invalid format, or has" From 3cfae50de0c3065c9d07b0fc1ac2416a28e0dabc Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 22 Feb 2022 10:00:54 -0600 Subject: [PATCH 02/14] install vim on docker build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fc9ddfe7b8..5613a1786b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ RUN apt-get update \ libmcrypt4 libmhash2 mcrypt \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ + && apt-get install vim RUN mkdir -p /var/www/$appname \ && mkdir -p /var/www/.cache/Python-Eggs/ \ From bce50df7a4204dbbcbcee8aed32e088bd90e2230 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 22 Feb 2022 12:24:44 -0600 Subject: [PATCH 03/14] fix docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5613a1786b..3c1a889b68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends curl bash git \ libmcrypt4 libmhash2 mcrypt \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/ + && rm -rf /var/lib/apt/lists/ \ && apt-get install vim RUN mkdir -p /var/www/$appname \ From d681451cc5250934e94f2618937ee16aaf2e96b3 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 22 Feb 2022 12:28:54 -0600 Subject: [PATCH 04/14] black --- fence/sync/sync_users.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index a29d20fcdf..22ca1acaf6 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -493,7 +493,9 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): if os.stat(filepath).st_size == 0: self.logger.warning("Empty file {}".format(filepath)) continue - if not self._match_pattern(filepath, custom_id_patterns=custom_id_patterns, encrypted=encrypted): + if not self._match_pattern( + filepath, custom_id_patterns=custom_id_patterns, encrypted=encrypted + ): self.logger.warning( "Filename {} does not match dbgap access control filename pattern;" " this could mean that the filename has an invalid format, or has" From 0fa758f1da7f4925fb2b18a201143620f88a88f4 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 22 Feb 2022 14:18:43 -0600 Subject: [PATCH 05/14] fix vim install --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3c1a889b68..3b11a8a466 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,10 +9,10 @@ RUN pip install --upgrade pip RUN pip install --upgrade poetry RUN apt-get update \ && apt-get install -y --no-install-recommends curl bash git \ + && apt-get -y install vim \ libmcrypt4 libmhash2 mcrypt \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/ \ - && apt-get install vim + && rm -rf /var/lib/apt/lists/ RUN mkdir -p /var/www/$appname \ && mkdir -p /var/www/.cache/Python-Eggs/ \ From 371ace16e3770c00f24d662e300d5f67328d78e6 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 1 Mar 2022 13:42:13 -0600 Subject: [PATCH 06/14] encrypted --- fence/sync/sync_users.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 22ca1acaf6..0558cd73b6 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1090,6 +1090,7 @@ def _process_dbgap_files(self, dbgap_config, sess): dbgap_file_list = [] hostname = dbgap_config["info"]["host"] username = dbgap_config["info"]["username"] + encrypted = dbgap_config["ingo"].get("encrypted", True) folderdir = os.path.join(str(self.folder), str(hostname), str(username)) try: @@ -1098,13 +1099,14 @@ def _process_dbgap_files(self, dbgap_config, sess): os.path.join(folderdir, "*") ) # get lists of file from folder else: + self.logger.info("Downloading files from: {}".format()) dbgap_file_list = self._download(dbgap_config) except Exception as e: self.logger.error(e) exit(1) self.logger.info("dbgap files: {}".format(dbgap_file_list)) user_projects, user_info = self._get_user_permissions_from_csv_list( - dbgap_file_list, encrypted=True, session=sess, dbgap_config=dbgap_config + dbgap_file_list, encrypted=encrypted, session=sess, dbgap_config=dbgap_config ) user_projects = self.parse_projects(user_projects) From c81e2a9d5750f8319f5070a0dc4e1fd5ae5fd199 Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 3 Mar 2022 11:34:55 -0600 Subject: [PATCH 07/14] fix typo --- fence/sync/sync_users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 0558cd73b6..e03c413228 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1090,7 +1090,7 @@ def _process_dbgap_files(self, dbgap_config, sess): dbgap_file_list = [] hostname = dbgap_config["info"]["host"] username = dbgap_config["info"]["username"] - encrypted = dbgap_config["ingo"].get("encrypted", True) + encrypted = dbgap_config["info"].get("encrypted", True) folderdir = os.path.join(str(self.folder), str(hostname), str(username)) try: From 0cdf78c19f86518c3b79ff6c3a709af14d49805c Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 3 Mar 2022 14:31:20 -0600 Subject: [PATCH 08/14] fix logs --- fence/sync/sync_users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index e03c413228..c702952eb8 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1099,7 +1099,7 @@ def _process_dbgap_files(self, dbgap_config, sess): os.path.join(folderdir, "*") ) # get lists of file from folder else: - self.logger.info("Downloading files from: {}".format()) + self.logger.info("Downloading files from: {}".format(hostname)) dbgap_file_list = self._download(dbgap_config) except Exception as e: self.logger.error(e) From 0437d336651816f862384bc409a2d684848702e3 Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 17 Mar 2022 14:45:32 -0500 Subject: [PATCH 09/14] add test --- fence/sync/sync_users.py | 22 ++- .../csv/authentication_file_PROJECT-12345.csv | 4 + tests/dbgap_sync/test_user_sync.py | 179 ++++++++++++------ tests/test-fence-config.yaml | 4 + 4 files changed, 150 insertions(+), 59 deletions(-) create mode 100644 tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index c702952eb8..d489947fbb 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -329,6 +329,9 @@ def __init__( self.folder = folder self.sync_from_visas = sync_from_visas self.fallback_to_dbgap_sftp = fallback_to_dbgap_sftp + self.allow_non_dbgap_whitelist = dbGaP[0].get( + "allow_non_dbgap_whitelist", False + ) self.auth_source = defaultdict(set) # auth_source used for logging. username : [source1, source2] @@ -340,7 +343,7 @@ def __init__( ) @staticmethod - def _match_pattern(filepath, custom_id_patterns, encrypted=True): + def _match_pattern(filepath, id_patterns, encrypted=True): """ Check if the filename matches dbgap access control file pattern @@ -351,8 +354,8 @@ def _match_pattern(filepath, custom_id_patterns, encrypted=True): Returns: bool: whether the pattern matches """ - custom_id_patterns.insert(0, "phs(\d{6})") - for pattern in custom_id_patterns: + id_patterns.insert(0, "phs(\d{6})") + for pattern in id_patterns: pattern = r"authentication_file_{}.(csv|txt)".format(pattern) if encrypted: pattern += ".enc" @@ -476,7 +479,11 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): # parse dbGaP sftp server information dbgap_key = dbgap_config.get("decrypt_key", None) parse_consent_code = dbgap_config.get("parse_consent_code", True) - custom_id_patterns = dbgap_config.get("allowed_custom_ids", []) + id_patterns = ( + dbgap_config.get("allowed_id_patterns", []) + if self.allow_non_dbgap_whitelist + else [] + ) enable_common_exchange_area_access = dbgap_config.get( "enable_common_exchange_area_access", False ) @@ -494,7 +501,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): self.logger.warning("Empty file {}".format(filepath)) continue if not self._match_pattern( - filepath, custom_id_patterns=custom_id_patterns, encrypted=encrypted + filepath, id_patterns=id_patterns, encrypted=encrypted ): self.logger.warning( "Filename {} does not match dbgap access control filename pattern;" @@ -1106,7 +1113,10 @@ def _process_dbgap_files(self, dbgap_config, sess): exit(1) self.logger.info("dbgap files: {}".format(dbgap_file_list)) user_projects, user_info = self._get_user_permissions_from_csv_list( - dbgap_file_list, encrypted=encrypted, session=sess, dbgap_config=dbgap_config + dbgap_file_list, + encrypted=encrypted, + session=sess, + dbgap_config=dbgap_config, ) user_projects = self.parse_projects(user_projects) diff --git a/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv b/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv new file mode 100644 index 0000000000..b712066728 --- /dev/null +++ b/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv @@ -0,0 +1,4 @@ +user name, login, phsid, +USER D,TESTUSERD,PROJECT-12345 +USER B,TESTUSERB,PROJECT-12345 +USER C,USERC,PROJECT-12345 \ No newline at end of file diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index 0be912a2c9..9d17e5ca85 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -62,16 +62,23 @@ def test_sync_incorrect_user_yaml_file(syncer, monkeypatch, db_session): assert syncer.arborist_client.create_policy.not_called() +@pytest.mark.parametrize("allow_non_dbgap_whitelist", [False, True]) @pytest.mark.parametrize("syncer", ["google", "cleversafe"], indirect=True) @pytest.mark.parametrize("parse_consent_code_config", [False, True]) def test_sync( - syncer, db_session, storage_client, parse_consent_code_config, monkeypatch + syncer, + db_session, + allow_non_dbgap_whitelist, + storage_client, + parse_consent_code_config, + monkeypatch, ): # patch the sync to use the parameterized config value monkeypatch.setitem( syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config ) monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) + monkeypatch.setattr(syncer, "allow_non_dbgap_whitelist", allow_non_dbgap_whitelist) syncer.sync() @@ -79,63 +86,129 @@ def test_sync( assert len(users) == 14 if parse_consent_code_config: - user = models.query_for_user(session=db_session, username="USERC") - assert equal_project_access( - user.project_access, - { - "phs000178.c1": ["read", "read-storage"], - "phs000178.c2": ["read", "read-storage"], - "phs000178.c999": ["read", "read-storage"], - "phs000179.c1": ["read", "read-storage"], - }, - ) + if allow_non_dbgap_whitelist: + user = models.query_for_user(session=db_session, username="TESTUSERD") + assert equal_project_access( + user.project_access, + { + "phs000179.c1": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + }, + ) - user = models.query_for_user(session=db_session, username="USERF") - assert equal_project_access( - user.project_access, - { - "phs000178.c1": ["read", "read-storage"], - "phs000178.c2": ["read", "read-storage"], - }, - ) + user = models.query_for_user(session=db_session, username="TESTUSERB") + assert equal_project_access( + user.project_access, + { + "phs000178.c1": ["read", "read-storage"], + "phs000179.c1": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + }, + ) - user = models.query_for_user(session=db_session, username="TESTUSERB") - assert equal_project_access( - user.project_access, - { - "phs000179.c1": ["read", "read-storage"], - "phs000178.c1": ["read", "read-storage"], - }, - ) + user = models.query_for_user(session=db_session, username="USERC") + assert equal_project_access( + user.project_access, + { + "phs000178.c1": ["read", "read-storage"], + "phs000178.c2": ["read", "read-storage"], + "phs000178.c999": ["read", "read-storage"], + "phs000179.c1": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + }, + ) + else: + user = models.query_for_user(session=db_session, username="USERC") + assert equal_project_access( + user.project_access, + { + "phs000178.c1": ["read", "read-storage"], + "phs000178.c2": ["read", "read-storage"], + "phs000178.c999": ["read", "read-storage"], + "phs000179.c1": ["read", "read-storage"], + }, + ) + + user = models.query_for_user(session=db_session, username="USERF") + assert equal_project_access( + user.project_access, + { + "phs000178.c1": ["read", "read-storage"], + "phs000178.c2": ["read", "read-storage"], + }, + ) + + user = models.query_for_user(session=db_session, username="TESTUSERB") + assert equal_project_access( + user.project_access, + { + "phs000179.c1": ["read", "read-storage"], + "phs000178.c1": ["read", "read-storage"], + }, + ) else: - user = models.query_for_user(session=db_session, username="USERC") - assert equal_project_access( - user.project_access, - { - "phs000178": ["read", "read-storage"], - "TCGA-PCAWG": ["read", "read-storage"], - "phs000179": ["read", "read-storage"], - }, - ) + if allow_non_dbgap_whitelist: + user = models.query_for_user(session=db_session, username="TESTUSERD") + assert equal_project_access( + user.project_access, + { + "phs000179": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + }, + ) - user = models.query_for_user(session=db_session, username="USERF") - assert equal_project_access( - user.project_access, - { - "phs000178": ["read", "read-storage"], - "TCGA-PCAWG": ["read", "read-storage"], - }, - ) + user = models.query_for_user(session=db_session, username="TESTUSERB") + assert equal_project_access( + user.project_access, + { + "phs000178": ["read", "read-storage"], + "phs000179": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + "TCGA-PCAWG": ["read", "read-storage"], + }, + ) - user = models.query_for_user(session=db_session, username="TESTUSERB") - assert equal_project_access( - user.project_access, - { - "phs000178": ["read", "read-storage"], - "TCGA-PCAWG": ["read", "read-storage"], - "phs000179": ["read", "read-storage"], - }, - ) + user = models.query_for_user(session=db_session, username="USERC") + assert equal_project_access( + user.project_access, + { + "phs000178": ["read", "read-storage"], + "phs000178": ["read", "read-storage"], + "phs000178": ["read", "read-storage"], + "phs000179": ["read", "read-storage"], + "TCGA-PCAWG": ["read", "read-storage"], + "PROJECT-12345": ["read", "read-storage"], + }, + ) + else: + user = models.query_for_user(session=db_session, username="USERC") + assert equal_project_access( + user.project_access, + { + "phs000178": ["read", "read-storage"], + "TCGA-PCAWG": ["read", "read-storage"], + "phs000179": ["read", "read-storage"], + }, + ) + + user = models.query_for_user(session=db_session, username="USERF") + assert equal_project_access( + user.project_access, + { + "phs000178": ["read", "read-storage"], + "TCGA-PCAWG": ["read", "read-storage"], + }, + ) + + user = models.query_for_user(session=db_session, username="TESTUSERB") + assert equal_project_access( + user.project_access, + { + "phs000178": ["read", "read-storage"], + "TCGA-PCAWG": ["read", "read-storage"], + "phs000179": ["read", "read-storage"], + }, + ) user = models.query_for_user(session=db_session, username="TESTUSERD") assert user.display_name == "USER D" diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml index 7b4535327b..a0692a2170 100755 --- a/tests/test-fence-config.yaml +++ b/tests/test-fence-config.yaml @@ -322,6 +322,8 @@ dbGaP: proxy_user: '' protocol: 'sftp' decrypt_key: '' + allow_non_dbGaP_whitelist: false + allowed_id_patterns: ['PROJECT-12345'] # parse out the consent from the dbgap accession number such that something # like "phs000123.v1.p1.c2" becomes "phs000123.c2". # @@ -349,6 +351,8 @@ dbGaP: 'phs000178': ['/orgA/', '/orgB/', '/'] # study when parsing consent codes 'phs000178.c2': ['/orgA/', '/orgB/', '/'] + # non dbgap study: + 'PROJECT-12345': [''] - info: host: '' username: '' From 9c9004326e598dfe7028dd9f19c77b6cf54cf8be Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 24 Mar 2022 13:11:13 -0500 Subject: [PATCH 10/14] change phsid to project_id --- fence/sync/sync_users.py | 2 +- tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index d489947fbb..b67c618ea4 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -523,7 +523,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): continue phsid_privileges = {} - phsid = row.get("phsid", "").split(".") + phsid = row.get("phsid", row.get("project_id", "")).split(".") dbgap_project = phsid[0] if len(phsid) > 1 and parse_consent_code: consent_code = phsid[-1] diff --git a/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv b/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv index b712066728..63f08761ae 100644 --- a/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv +++ b/tests/dbgap_sync/data/csv/authentication_file_PROJECT-12345.csv @@ -1,4 +1,4 @@ -user name, login, phsid, +user name, login, project_id USER D,TESTUSERD,PROJECT-12345 USER B,TESTUSERB,PROJECT-12345 USER C,USERC,PROJECT-12345 \ No newline at end of file From eb16450a525443a9cf474f0d259cfe006401d2ad Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 9 May 2022 11:36:32 -0500 Subject: [PATCH 11/14] Resolve Reivew --- fence/config-default.yaml | 2 -- fence/sync/sync_users.py | 26 ++++++++++++++++---------- tests/dbgap_sync/test_user_sync.py | 9 +++++++-- tests/test-fence-config.yaml | 27 +++++++++++++++++++++++---- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 34f5ed563c..7c7fdf4456 100755 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -904,8 +904,6 @@ SERVICE_ACCOUNT_LIMIT: 6 GLOBAL_PARSE_VISAS_ON_LOGIN: # Settings for usersync with visas USERSYNC: - # allowed list of custom project ids that can be processed by usersync - allowed_custom_project_ids: ["CUSTOM-*"] sync_from_visas: false # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed fallback_to_dbgap_sftp: false diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index b67c618ea4..24c77fd435 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -329,9 +329,6 @@ def __init__( self.folder = folder self.sync_from_visas = sync_from_visas self.fallback_to_dbgap_sftp = fallback_to_dbgap_sftp - self.allow_non_dbgap_whitelist = dbGaP[0].get( - "allow_non_dbgap_whitelist", False - ) self.auth_source = defaultdict(set) # auth_source used for logging. username : [source1, source2] @@ -341,6 +338,10 @@ def __init__( self.storage_manager = StorageManager( storage_credentials, logger=self.logger ) + self.id_patterns = [] + self.non_dbGaP_whitelist = config.get("USERSYNC", {}).get( + "non_dbgap_whitelist_enabled", False + ) @staticmethod def _match_pattern(filepath, id_patterns, encrypted=True): @@ -354,9 +355,9 @@ def _match_pattern(filepath, id_patterns, encrypted=True): Returns: bool: whether the pattern matches """ - id_patterns.insert(0, "phs(\d{6})") + id_patterns.append("authentication_file_phs(\d{6}).(csv|txt)") for pattern in id_patterns: - pattern = r"authentication_file_{}.(csv|txt)".format(pattern) + pattern = r"{}".format(pattern) if encrypted: pattern += ".enc" pattern += "$" @@ -479,9 +480,10 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): # parse dbGaP sftp server information dbgap_key = dbgap_config.get("decrypt_key", None) parse_consent_code = dbgap_config.get("parse_consent_code", True) - id_patterns = ( - dbgap_config.get("allowed_id_patterns", []) - if self.allow_non_dbgap_whitelist + self.id_patterns += ( + dbgap_config.get("allowed_whitelist_patterns", []) + if self.non_dbGaP_whitelist + and dbgap_config.get("allow_non_dbGaP_whitelist", False) else [] ) enable_common_exchange_area_access = dbgap_config.get( @@ -501,7 +503,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): self.logger.warning("Empty file {}".format(filepath)) continue if not self._match_pattern( - filepath, id_patterns=id_patterns, encrypted=encrypted + filepath, id_patterns=self.id_patterns, encrypted=encrypted ): self.logger.warning( "Filename {} does not match dbgap access control filename pattern;" @@ -523,7 +525,11 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): continue phsid_privileges = {} - phsid = row.get("phsid", row.get("project_id", "")).split(".") + if self.non_dbGaP_whitelist: + phsid = row.get("phsid", row.get("project_id", "")).split(".") + else: + phsid = row.get("phsid", "").split(".") + dbgap_project = phsid[0] if len(phsid) > 1 and parse_consent_code: consent_code = phsid[-1] diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index 9d17e5ca85..54569267bd 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -78,7 +78,12 @@ def test_sync( syncer.dbGaP[0], "parse_consent_code", parse_consent_code_config ) monkeypatch.setattr(syncer, "parse_consent_code", parse_consent_code_config) - monkeypatch.setattr(syncer, "allow_non_dbgap_whitelist", allow_non_dbgap_whitelist) + monkeypatch.setitem( + syncer.dbGaP[2], "allow_non_dbGaP_whitelist", allow_non_dbgap_whitelist + ) + monkeypatch.setattr(syncer, "non_dbGaP_whitelist", allow_non_dbgap_whitelist) + # merge sftp server containing both dbgap and non-dbgap whitelists + monkeypatch.setattr(syncer, "is_sync_from_dbgap_server", allow_non_dbgap_whitelist) syncer.sync() @@ -683,7 +688,7 @@ def mock_merge(dbgap_servers, sess): # this function will be called once for each sftp server # the test config file has 2 dbgap sftp servers - assert syncer._process_dbgap_files.call_count == 2 + assert syncer._process_dbgap_files.call_count == 3 @pytest.mark.parametrize("syncer", ["cleversafe", "google"], indirect=True) diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml index f530e91d1d..495e3b23f5 100755 --- a/tests/test-fence-config.yaml +++ b/tests/test-fence-config.yaml @@ -341,8 +341,6 @@ dbGaP: proxy_user: '' protocol: 'sftp' decrypt_key: '' - allow_non_dbGaP_whitelist: false - allowed_id_patterns: ['PROJECT-12345'] # parse out the consent from the dbgap accession number such that something # like "phs000123.v1.p1.c2" becomes "phs000123.c2". # @@ -370,8 +368,6 @@ dbGaP: 'phs000178': ['/orgA/', '/orgB/', '/'] # study when parsing consent codes 'phs000178.c2': ['/orgA/', '/orgB/', '/'] - # non dbgap study: - 'PROJECT-12345': [''] - info: host: '' username: '' @@ -408,6 +404,20 @@ dbGaP: 'phs000178': ['/orgA/', '/orgB/', '/'] # study when parsing consent codes 'phs000178.c2': ['/orgA/', '/orgB/', '/'] + - info: + host: '' + username: '' + password: '' + port: 22 + proxy: '' + proxy_user: '' + protocol: 'sftp' + decrypt_key: '' + allow_non_dbGaP_whitelist: true + allowed_whitelist_patterns: ['authentication_file_PROJECT-(\d*).(csv|txt)'] + study_to_resource_namespaces: + # non dbgap study: + 'PROJECT-12345': [''] # Regex to match an assession number that has consent information in forms like: # phs00301123.c999 # phs000123.v3.p1.c3 @@ -639,3 +649,12 @@ ASSUME_ROLE_CACHE_SECONDS: 1800 # List of JWT issuers from which Fence will accept GA4GH visas GA4GH_VISA_ISSUER_ALLOWLIST: - 'https://stsstg.nih.gov' + +USERSYNC: + non_dbgap_whitelist_enabled: false + sync_from_visas: false + # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed + fallback_to_dbgap_sftp: false + visa_types: + ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] +RAS_USERINFO_ENDPOINT: '/openid/connect/v1.1/userinfo' From 85bb78576254f9eabf075db48a856a006059a29c Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 1 Jul 2022 12:02:09 -0500 Subject: [PATCH 12/14] Fix all the things --- fence/sync/sync_users.py | 38 ++++++--- poetry.lock | 122 ++++++++++++++++------------- tests/dbgap_sync/test_user_sync.py | 9 +-- tests/test-fence-config.yaml | 3 +- 4 files changed, 97 insertions(+), 75 deletions(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index bc1efaa36c..c4918a30d5 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -357,9 +357,6 @@ def __init__( storage_credentials, logger=self.logger ) self.id_patterns = [] - self.non_dbGaP_whitelist = config.get("USERSYNC", {}).get( - "non_dbgap_whitelist_enabled", False - ) @staticmethod def _match_pattern(filepath, id_patterns, encrypted=True): @@ -497,11 +494,10 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): # parse dbGaP sftp server information dbgap_key = dbgap_config.get("decrypt_key", None) - parse_consent_code = dbgap_config.get("parse_consent_code", True) + self.id_patterns += ( dbgap_config.get("allowed_whitelist_patterns", []) - if self.non_dbGaP_whitelist - and dbgap_config.get("allow_non_dbGaP_whitelist", False) + if dbgap_config.get("allow_non_dbGaP_whitelist", False) else [] ) enable_common_exchange_area_access = dbgap_config.get( @@ -511,7 +507,7 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): "study_common_exchange_areas", {} ) - if parse_consent_code and enable_common_exchange_area_access: + if self.parse_consent_code and enable_common_exchange_area_access: self.logger.info( f"using study to common exchange area mapping: {study_common_exchange_areas}" ) @@ -543,13 +539,13 @@ def _parse_csv(self, file_dict, sess, dbgap_config={}, encrypted=True): continue phsid_privileges = {} - if self.non_dbGaP_whitelist: + if dbgap_config.get("allow_non_dbGaP_whitelist", False): phsid = row.get("phsid", row.get("project_id", "")).split(".") else: phsid = row.get("phsid", "").split(".") dbgap_project = phsid[0] - if len(phsid) > 1 and parse_consent_code: + if len(phsid) > 1 and self.parse_consent_code: consent_code = phsid[-1] # c999 indicates full access to all consents and access @@ -1261,6 +1257,24 @@ def _get_user_permissions_from_csv_list( ) return user_projects, user_info + def _merge_multiple_local_csv_files( + self, dbgap_file_list, encrypted, dbgap_configs, session + ): + + merged_user_projects = {} + merged_user_info = {} + + for dbgap_config in dbgap_configs: + user_projects, user_info = self._get_user_permissions_from_csv_list( + dbgap_file_list, + encrypted, + session=session, + dbgap_config=dbgap_config, + ) + self.sync_two_user_info_dict(user_info, merged_user_info) + self.sync_two_phsids_dict(user_projects, merged_user_projects) + return merged_user_projects, merged_user_info + def _merge_multiple_dbgap_sftp(self, dbgap_servers, sess): """ Args: @@ -1442,13 +1456,11 @@ def _sync(self, sess): os.path.join(self.sync_from_local_csv_dir, "*") ) - # if syncing from local csv dir dbgap configurations - # come from the first dbgap instance in the fence config file - user_projects_csv, user_info_csv = self._get_user_permissions_from_csv_list( + user_projects_csv, user_info_csv = self._merge_multiple_local_csv_files( local_csv_file_list, encrypted=False, session=sess, - dbgap_config=self.dbGaP[0], + dbgap_configs=self.dbGaP, ) try: diff --git a/poetry.lock b/poetry.lock index 53de10e81e..5d58cc97e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -81,7 +81,7 @@ fastapi = ["fastapi (>=0.54.1,<0.55.0)"] [[package]] name = "azure-core" -version = "1.24.1" +version = "1.24.2" description = "Microsoft Azure Core Library for Python" category = "main" optional = false @@ -252,7 +252,7 @@ python-versions = ">=3.6" [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -1538,8 +1538,8 @@ authutils = [ {file = "authutils-6.1.0.tar.gz", hash = "sha256:7263af0b2ce3a0db19236fd123b34f795d07e07111b7bd18a51808568ddfdc2e"}, ] azure-core = [ - {file = "azure-core-1.24.1.zip", hash = "sha256:39c5d59d04209bb70a1a7ee879cef05d07bc76472cd3fb5eaa2e607a90d312bb"}, - {file = "azure_core-1.24.1-py3-none-any.whl", hash = "sha256:f48a640affa59fa45ac770565b3bead4c4f834242d16983c1ae2bb173a4b8a6d"}, + {file = "azure-core-1.24.2.zip", hash = "sha256:0f3a20d245659bf81fb3670070a5410c8d4a43298d5a981e62dce393000a9084"}, + {file = "azure_core-1.24.2-py3-none-any.whl", hash = "sha256:a76856fa83efe1925a4fd917dc179c7daa15917dd71da2774833fa82a96f3dfa"}, ] azure-storage-blob = [ {file = "azure-storage-blob-12.12.0.zip", hash = "sha256:f6daf07d1ca86d189ae15c9b1859dff5b7127bf24a07a4bbe41e0b81e01d62f7"}, @@ -1602,56 +1602,70 @@ certifi = [ {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cffi = [ - {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, - {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, - {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, - {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, - {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, - {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, - {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, - {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, - {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, - {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, - {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, - {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, - {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, - {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, - {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, - {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, - {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, ] charset-normalizer = [ {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index b7d406977f..6b8ca093f2 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -74,9 +74,9 @@ def test_sync_incorrect_user_yaml_file(syncer, monkeypatch, db_session): assert syncer.arborist_client.create_policy.not_called() -@pytest.mark.parametrize("allow_non_dbgap_whitelist", [False, True]) +@pytest.mark.parametrize("allow_non_dbgap_whitelist", [True, False]) @pytest.mark.parametrize("syncer", ["google", "cleversafe"], indirect=True) -@pytest.mark.parametrize("parse_consent_code_config", [False, True]) +@pytest.mark.parametrize("parse_consent_code_config", [True, False]) def test_sync( syncer, db_session, @@ -93,9 +93,6 @@ def test_sync( monkeypatch.setitem( syncer.dbGaP[2], "allow_non_dbGaP_whitelist", allow_non_dbgap_whitelist ) - monkeypatch.setattr(syncer, "non_dbGaP_whitelist", allow_non_dbgap_whitelist) - # merge sftp server containing both dbgap and non-dbgap whitelists - monkeypatch.setattr(syncer, "is_sync_from_dbgap_server", allow_non_dbgap_whitelist) syncer.sync() @@ -710,7 +707,7 @@ def mock_merge(dbgap_servers, sess): syncer.sync() # this function will be called once for each sftp server - # the test config file has 2 dbgap sftp servers + # the test config file has 3 dbgap sftp servers assert syncer._process_dbgap_files.call_count == 3 diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml index d29653246b..9ed477e79c 100755 --- a/tests/test-fence-config.yaml +++ b/tests/test-fence-config.yaml @@ -413,7 +413,7 @@ dbGaP: proxy_user: '' protocol: 'sftp' decrypt_key: '' - allow_non_dbGaP_whitelist: true + allow_non_dbGaP_whitelist: false allowed_whitelist_patterns: ['authentication_file_PROJECT-(\d*).(csv|txt)'] study_to_resource_namespaces: # non dbgap study: @@ -676,7 +676,6 @@ EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS: 1 GLOBAL_PARSE_VISAS_ON_LOGIN: # Settings for usersync with visas USERSYNC: - non_dbgap_whitelist_enabled: false sync_from_visas: false # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed fallback_to_dbgap_sftp: false From 1afd7c2d9a4a6e297855d9cb2037f0225211b096 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 5 Jul 2022 16:48:10 -0500 Subject: [PATCH 13/14] separate basename --- fence/sync/sync_users.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index c4918a30d5..4faa258e5d 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -376,7 +376,8 @@ def _match_pattern(filepath, id_patterns, encrypted=True): if encrypted: pattern += ".enc" pattern += "$" - if re.match(pattern, os.path.basename(filepath)): + basename = os.path.basename(filepath) + if re.match(pattern, basename): return True return False From 0aa3d456f92397fe70ee80caa2f8558b5182af44 Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 6 Jul 2022 15:04:11 -0500 Subject: [PATCH 14/14] small changes --- fence/sync/sync_users.py | 3 +-- tests/dbgap_sync/test_user_sync.py | 4 ++-- tests/test-fence-config.yaml | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 4faa258e5d..c4918a30d5 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -376,8 +376,7 @@ def _match_pattern(filepath, id_patterns, encrypted=True): if encrypted: pattern += ".enc" pattern += "$" - basename = os.path.basename(filepath) - if re.match(pattern, basename): + if re.match(pattern, os.path.basename(filepath)): return True return False diff --git a/tests/dbgap_sync/test_user_sync.py b/tests/dbgap_sync/test_user_sync.py index 6b8ca093f2..3eaf12f054 100644 --- a/tests/dbgap_sync/test_user_sync.py +++ b/tests/dbgap_sync/test_user_sync.py @@ -74,9 +74,9 @@ def test_sync_incorrect_user_yaml_file(syncer, monkeypatch, db_session): assert syncer.arborist_client.create_policy.not_called() -@pytest.mark.parametrize("allow_non_dbgap_whitelist", [True, False]) +@pytest.mark.parametrize("allow_non_dbgap_whitelist", [False, True]) @pytest.mark.parametrize("syncer", ["google", "cleversafe"], indirect=True) -@pytest.mark.parametrize("parse_consent_code_config", [True, False]) +@pytest.mark.parametrize("parse_consent_code_config", [False, True]) def test_sync( syncer, db_session, diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml index 9ed477e79c..33b6bb9a5b 100755 --- a/tests/test-fence-config.yaml +++ b/tests/test-fence-config.yaml @@ -676,9 +676,6 @@ EXPIRED_AUTHZ_REMOVAL_JOB_FREQ_IN_SECONDS: 1 GLOBAL_PARSE_VISAS_ON_LOGIN: # Settings for usersync with visas USERSYNC: - sync_from_visas: false - # fallback to dbgap sftp when there are no valid visas for a user i.e. if they're expired or if they're malformed - fallback_to_dbgap_sftp: false visa_types: ras: ["https://ras.nih.gov/visas/v1", "https://ras.nih.gov/visas/v1.1"] RAS_USERINFO_ENDPOINT: '/openid/connect/v1.1/userinfo'