From 5d4aba296de5629837d698aa45098fbe92153899 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Mon, 14 Oct 2019 12:28:10 -0500 Subject: [PATCH 1/4] fix(usersync): get list of users from arborist and combine with users from authz sources --- fence/sync/sync_users.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 5ee381897c..38404178b0 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1148,6 +1148,27 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): # update the project info with `projects` specified in user.yaml self.sync_two_phsids_dict(user_yaml.user_abac, user_projects) + # get list of users from arborist to make sure users that are completely removed + # from authorization sources get policies revoked + arborist_users = {} + try: + arborist_users = self.arborist_client.get(url="/user").json + except ArboristError as error: + self.logger.warning( + "Could not get list of users in Arborist, continuing anyway. " + "WARNING: this sync will NOT remove access for users no longer in " + f"authorization sources. Arborist error: {error}" + ) + # TODO usersync should exit with non-zero exit code at the end, but sync + # itself should continue + + arborist_user_projects = { + user.get("name", "unknown"): {} for user in arborist_users.items() + } + + # update the project info with users from arborist + self.sync_two_phsids_dict(arborist_user_projects, user_projects) + for username, user_project_info in user_projects.items(): self.logger.info("processing user `{}`".format(username)) user = query_for_user(session=session, username=username) From 9869103f2e1300ee3513cc78606b91afae4b0ef5 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Mon, 14 Oct 2019 13:11:32 -0500 Subject: [PATCH 2/4] fix(usersync): hit correct arborist url for listing users --- fence/sync/sync_users.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 38404178b0..4140f8c170 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1152,7 +1152,9 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): # from authorization sources get policies revoked arborist_users = {} try: - arborist_users = self.arborist_client.get(url="/user").json + arborist_users = self.arborist_client.get( + url=self.arborist_client._user_url + ).json except ArboristError as error: self.logger.warning( "Could not get list of users in Arborist, continuing anyway. " From 3de5c56148d7c77629cd1280c88e6bb55add1688 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Mon, 14 Oct 2019 13:25:14 -0500 Subject: [PATCH 3/4] fix(tests): add arborist client attribute to magic mock for testing --- tests/dbgap_sync/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/dbgap_sync/conftest.py b/tests/dbgap_sync/conftest.py index 3f3a26e814..43ed2a4232 100644 --- a/tests/dbgap_sync/conftest.py +++ b/tests/dbgap_sync/conftest.py @@ -156,6 +156,8 @@ def mocked_update(parent_path, resource, **kwargs): syncer_obj.arborist_client.get_policy.side_effect = lambda _: None + syncer_obj.arborist_client._user_url = "/user" + for element in provider: udm.create_provider(db_session, element["name"], backend=element["backend"]) From 8f7eed122196ed3b0422df09b4f610300a8e4fe3 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Mon, 14 Oct 2019 13:33:14 -0500 Subject: [PATCH 4/4] fix(usersync): adjust logic to construct correct dict from arborist response and log errors --- fence/sync/sync_users.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 4140f8c170..634d20ca9c 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1150,23 +1150,27 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): # get list of users from arborist to make sure users that are completely removed # from authorization sources get policies revoked - arborist_users = {} + arborist_user_projects = {} try: arborist_users = self.arborist_client.get( url=self.arborist_client._user_url - ).json - except ArboristError as error: + ).json["users"] + + # construct user information, NOTE the lowering of the username. when adding/ + # removing access, the case in the Fence db is used. For combining access, it is + # case-insensitive, so we lower + arborist_user_projects = { + user["name"].lower(): {} for user in arborist_users.items() + } + except (ArboristError, KeyError) as error: + # TODO usersync should probably exit with non-zero exit code at the end, + # but sync should continue from this point so there are no partial + # updates self.logger.warning( "Could not get list of users in Arborist, continuing anyway. " "WARNING: this sync will NOT remove access for users no longer in " - f"authorization sources. Arborist error: {error}" + f"authorization sources. Error: {error}" ) - # TODO usersync should exit with non-zero exit code at the end, but sync - # itself should continue - - arborist_user_projects = { - user.get("name", "unknown"): {} for user in arborist_users.items() - } # update the project info with users from arborist self.sync_two_phsids_dict(arborist_user_projects, user_projects)