From 895bf3964e93f3402edba67fc302d0964bc5528e Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 15 Aug 2019 17:16:10 -0500 Subject: [PATCH 01/15] feat(synapse): support Synapse as OIDC provider --- fence/__init__.py | 10 +++ fence/blueprints/login/__init__.py | 8 +++ fence/blueprints/login/base.py | 7 ++- fence/blueprints/login/synapse.py | 32 ++++++++++ fence/resources/openid/synapse_oauth2.py | 80 ++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 fence/blueprints/login/synapse.py create mode 100644 fence/resources/openid/synapse_oauth2.py diff --git a/fence/__init__.py b/fence/__init__.py index 96937a5a58..ea421643e1 100644 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -20,6 +20,7 @@ MicrosoftOauth2Client as MicrosoftClient, ) from fence.resources.openid.orcid_oauth2 import OrcidOauth2Client as ORCIDClient +from fence.resources.openid.synapse_oauth2 import SynapseOauth2Client as SynapseClient from fence.resources.storage import StorageManager from fence.resources.user.user_session import UserSessionInterface from fence.error_handler import get_error_response @@ -248,6 +249,7 @@ def _set_authlib_cfgs(app): def _setup_oidc_clients(app): enabled_idp_ids = list(config["ENABLED_IDENTITY_PROVIDERS"]["providers"].keys()) + oidc = config.get("OPENID_CONNECT", {}) # Add OIDC client for Google if configured. configured_google = ( @@ -271,6 +273,14 @@ def _setup_oidc_clients(app): logger=logger, ) + # Add OIDC client for Synapse if configured. + if 'synapse' in oidc: + app.synapse_client = SynapseClient( + oidc["synapse"], + HTTP_PROXY=config.get("HTTP_PROXY"), + logger=logger, + ) + # Add OIDC client for Microsoft if configured. configured_microsoft = ( "OPENID_CONNECT" in config and "microsoft" in config["OPENID_CONNECT"] diff --git a/fence/blueprints/login/__init__.py b/fence/blueprints/login/__init__.py index 1977d989a3..316fa0ea5f 100644 --- a/fence/blueprints/login/__init__.py +++ b/fence/blueprints/login/__init__.py @@ -13,6 +13,7 @@ from fence.blueprints.login.shib import ShibbolethLogin, ShibbolethCallback from fence.blueprints.login.microsoft import MicrosoftLogin, MicrosoftCallback from fence.blueprints.login.orcid import ORCIDLogin, ORCIDCallback +from fence.blueprints.login.synapse import SynapseLogin, SynapseCallback from fence.errors import InternalError from fence.restful import RestfulApi from fence.config import config @@ -27,6 +28,7 @@ "google": "google", "shibboleth": "shib", "orcid": "orcid", + "synapse": "synapse", "microsoft": "microsoft", } @@ -112,6 +114,12 @@ def provider_info(idp_id): blueprint_api.add_resource(ORCIDLogin, "/orcid", strict_slashes=False) blueprint_api.add_resource(ORCIDCallback, "/orcid/login", strict_slashes=False) + if "synapse" in idps: + blueprint_api.add_resource(SynapseLogin, "/synapse", strict_slashes=False) + blueprint_api.add_resource( + SynapseCallback, "/synapse/login", strict_slashes=False + ) + if "microsoft" in idps: blueprint_api.add_resource(MicrosoftLogin, "/microsoft", strict_slashes=False) blueprint_api.add_resource( diff --git a/fence/blueprints/login/base.py b/fence/blueprints/login/base.py index c05795d730..a41c08b6d2 100644 --- a/fence/blueprints/login/base.py +++ b/fence/blueprints/login/base.py @@ -73,9 +73,14 @@ def get(self): result = self.client.get_user_id(code) username = result.get(self.username_field) if username: - return _login(username, self.idp_name) + resp = _login(username, self.idp_name) + self.post_login(flask.g.user, result) + return resp raise UserError(result) + def post_login(self, user, token_result): + pass + def _login(username, idp_name): """ diff --git a/fence/blueprints/login/synapse.py b/fence/blueprints/login/synapse.py new file mode 100644 index 0000000000..83061fbef2 --- /dev/null +++ b/fence/blueprints/login/synapse.py @@ -0,0 +1,32 @@ +import flask +from flask_sqlalchemy_session import current_session + +from fence.models import IdentityProvider + +from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback + + +class SynapseLogin(DefaultOAuth2Login): + def __init__(self): + super(SynapseLogin, self).__init__( + idp_name=IdentityProvider.synapse, client=flask.current_app.synapse_client + ) + + +class SynapseCallback(DefaultOAuth2Callback): + def __init__(self): + super(SynapseCallback, self).__init__( + idp_name=IdentityProvider.synapse, + client=flask.current_app.synapse_client, + ) + + def post_login(self, user, token_result): + user.id_from_idp = token_result["sub"] + user.display_name = "{given_name} {family_name}".format(**token_result) + current_session.add(user) + current_session.commit() + + # TODO + # company + # team + # exp diff --git a/fence/resources/openid/synapse_oauth2.py b/fence/resources/openid/synapse_oauth2.py new file mode 100644 index 0000000000..7c57ed59f1 --- /dev/null +++ b/fence/resources/openid/synapse_oauth2.py @@ -0,0 +1,80 @@ +import json + +from .idp_oauth2 import Oauth2ClientBase + + +class SynapseOauth2Client(Oauth2ClientBase): + """ + client for interacting with Synapse OAuth2, + as OpenID Connect is supported under OAuth2 + + """ + + ISSUER = "https://www.synapse.org/auth/v1" + REQUIRED_CLAIMS = {"give_name", "family_name", "email"} + OPTIONAL_CLAIMS = {"company"} + SYSTEM_CLAIMS = {"sub", "exp"} + CUSTOM_CLAIMS = {"team"} + + def __init__(self, settings, logger, HTTP_PROXY=None): + super(SynapseOauth2Client, self).__init__( + settings, + logger, + scope="openid", + discovery_url=self.ISSUER + "/.well-known/openid-configuration", + idp="Synapse", + HTTP_PROXY=HTTP_PROXY, + ) + + def get_auth_url(self): + """ + Get authorization uri from discovery doc + """ + authorization_endpoint = self.get_value_from_discovery_doc( + "authorization_endpoint", self.ISSUER + "/oauth2/authorize" + ) + + claims = dict( + id_token=dict( + team=dict(values=["1"]), + **{ + claim: dict(essential=claim in self.REQUIRED_CLAIMS) + for claim in self.REQUIRED_CLAIMS | self.OPTIONAL_CLAIMS + } + ) + ) + uri, state = self.session.create_authorization_url( + authorization_endpoint, + prompt="login", + claims=json.dumps(claims, separators=(",", ": ")), + ) + + return uri + + def get_user_id(self, code): + try: + token_endpoint = self.get_value_from_discovery_doc( + "token_endpoint", self.ISSUER + "/oauth2/token" + ) + jwks_endpoint = self.get_value_from_discovery_doc( + "jwks_uri", self.ISSUER + "/oauth2/jwks" + ) + claims = self.get_jwt_claims_identity(token_endpoint, jwks_endpoint, code) + + if not claims["email_verified"]: + return dict(error="Email is not verified") + + rv = {} + for claim in ( + self.REQUIRED_CLAIMS + | self.OPTIONAL_CLAIMS + | self.SYSTEM_CLAIMS + | self.CUSTOM_CLAIMS + ): + if claim not in self.OPTIONAL_CLAIMS and claim not in claims: + return dict(error="Required claim {} not found".format(claim)) + rv[claim] = claims[claim] + return rv + except Exception as e: + self.logger.exception("Can't get user info") + return {"error": "Can't get ID token: {}".format(e)} From 5f49f2be3099ea3171031ed9def9c5c6f810195b Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 3 Sep 2019 14:34:23 -0500 Subject: [PATCH 02/15] feat(synapse): authz --- fence/blueprints/login/synapse.py | 18 ++++++++++++------ fence/config-default.yaml | 7 +++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/fence/blueprints/login/synapse.py b/fence/blueprints/login/synapse.py index 83061fbef2..8b67fbae67 100644 --- a/fence/blueprints/login/synapse.py +++ b/fence/blueprints/login/synapse.py @@ -1,6 +1,9 @@ +from datetime import datetime, timezone, timedelta + import flask from flask_sqlalchemy_session import current_session +from fence.config import config from fence.models import IdentityProvider from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback @@ -16,17 +19,20 @@ def __init__(self): class SynapseCallback(DefaultOAuth2Callback): def __init__(self): super(SynapseCallback, self).__init__( - idp_name=IdentityProvider.synapse, - client=flask.current_app.synapse_client, + idp_name=IdentityProvider.synapse, client=flask.current_app.synapse_client ) def post_login(self, user, token_result): user.id_from_idp = token_result["sub"] user.display_name = "{given_name} {family_name}".format(**token_result) + # TODO: token_result['company'] current_session.add(user) current_session.commit() - # TODO - # company - # team - # exp + if config["DREAM_CHALLENGE_TEAM"] in token_result.get("team", []): + flask.current_app.arborist.add_user_to_group( + user.username, + config["DREAM_CHALLENGE_GROUP"], + datetime.now(timezone.utc) + + timedelta(seconds=config["SYNAPSE_AUTHZ_TTL"]), + ) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 4fcd4f36f7..4b211b9652 100644 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -646,3 +646,10 @@ ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: - 'appspot.gserviceaccount.com' # user-managed service account - 'iam.gserviceaccount.com' + +# Synapse integration and DREAM challenge mapping. Team is from Synapse, and group is +# providing the actual permission in Arborist. User will be added to the group for TTL +# seconds if the team matches. +DREAM_CHALLENGE_TEAM: 'DREAM' +DREAM_CHALLENGE_GROUP: 'DREAM' +SYNAPSE_AUTHZ_TTL: 86400 \ No newline at end of file From f17235e03349db0ef29b79272e94e3059ab89810 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 10 Sep 2019 19:16:04 -0500 Subject: [PATCH 03/15] add auth_provider --- bin/fence-create | 1 + fence/blueprints/login/synapse.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/fence-create b/bin/fence-create index 6390a24d9a..389edb4440 100755 --- a/bin/fence-create +++ b/bin/fence-create @@ -354,6 +354,7 @@ def main(): arborist = ArboristClient( arborist_base_url=args.arborist, logger=get_logger("user_syncer.arborist_client"), + authz_provider="user-sync" ) if args.action == "create": diff --git a/fence/blueprints/login/synapse.py b/fence/blueprints/login/synapse.py index 8b67fbae67..3d03f737a1 100644 --- a/fence/blueprints/login/synapse.py +++ b/fence/blueprints/login/synapse.py @@ -29,10 +29,15 @@ def post_login(self, user, token_result): current_session.add(user) current_session.commit() - if config["DREAM_CHALLENGE_TEAM"] in token_result.get("team", []): - flask.current_app.arborist.add_user_to_group( - user.username, - config["DREAM_CHALLENGE_GROUP"], - datetime.now(timezone.utc) - + timedelta(seconds=config["SYNAPSE_AUTHZ_TTL"]), - ) + with flask.current_app.arborist.context(auth_provider="synapse"): + if config["DREAM_CHALLENGE_TEAM"] in token_result.get("team", []): + flask.current_app.arborist.add_user_to_group( + user.username, + config["DREAM_CHALLENGE_GROUP"], + datetime.now(timezone.utc) + + timedelta(seconds=config["SYNAPSE_AUTHZ_TTL"]), + ) + else: + flask.current_app.arborist.remove_user_from_group( + user.username, config["DREAM_CHALLENGE_GROUP"] + ) From c1daaaa4e8712d8e6b8556ccc28030192a86eca2 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Thu, 12 Sep 2019 10:49:11 -0500 Subject: [PATCH 04/15] use update_policy() --- fence/sync/sync_users.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 622032f01a..10f0a357f0 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1077,15 +1077,19 @@ def _update_arborist(self, session, user_yaml): policies = user_yaml.authz.get("policies", []) for policy in policies: + policy_id = policy.pop("id") + if not policy_id: + continue try: response = self.arborist_client.update_policy( - policy["id"], policy, create_if_not_exist=True + policy_id, policy, create_if_not_exist=True ) - if response: - self._created_policies.add(policy["id"]) except ArboristError as e: self.logger.error(e) # keep going; maybe just some conflicts from things existing already + else: + if response: + self._created_policies.add(policy_id) groups = user_yaml.authz.get("groups", []) for group in groups: @@ -1197,7 +1201,6 @@ def _update_authz_in_arborist(self, session, user_projects, user_yaml=None): self.arborist_client.update_policy( policy_id, { - "id": policy_id, "description": "policy created by fence sync", "role_ids": [permission], "resource_paths": [path], From d342e713399794e72150a5906f5915082ecdbb2c Mon Sep 17 00:00:00 2001 From: Fantix King Date: Mon, 16 Sep 2019 15:22:30 -0500 Subject: [PATCH 05/15] additional_info --- fence/blueprints/login/synapse.py | 4 +++- fence/models.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fence/blueprints/login/synapse.py b/fence/blueprints/login/synapse.py index 3d03f737a1..eab89dd2b3 100644 --- a/fence/blueprints/login/synapse.py +++ b/fence/blueprints/login/synapse.py @@ -25,7 +25,9 @@ def __init__(self): def post_login(self, user, token_result): user.id_from_idp = token_result["sub"] user.display_name = "{given_name} {family_name}".format(**token_result) - # TODO: token_result['company'] + if user.additional_info is None: + user.additional_info = {} + user.additional_info.update(token_result) current_session.add(user) current_session.commit() diff --git a/fence/models.py b/fence/models.py index d61c1e3c69..92f66c25d1 100644 --- a/fence/models.py +++ b/fence/models.py @@ -24,8 +24,8 @@ Text, MetaData, Table, -) -from sqlalchemy.dialects.postgresql import ARRAY + text) +from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.orm import relationship, backref from sqlalchemy.sql import func from sqlalchemy import exc as sa_exc @@ -689,6 +689,15 @@ def migrate(driver): metadata=md, ) + add_column_if_not_exist( + table_name=User.__tablename__, + column=Column( + "additional_info", JSONB(), server_default=text("'{}'") + ), + driver=driver, + metadata=md, + ) + def add_foreign_key_column_if_not_exist( table_name, From d7cfad9e47965d5e28aa72ffb4d5ded91656b42e Mon Sep 17 00:00:00 2001 From: Fantix King Date: Tue, 17 Sep 2019 19:58:12 -0400 Subject: [PATCH 06/15] user_audit_log --- fence/models.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fence/models.py b/fence/models.py index 92f66c25d1..3e586cbda5 100644 --- a/fence/models.py +++ b/fence/models.py @@ -698,6 +698,34 @@ def migrate(driver): metadata=md, ) + with driver.session as session: + session.execute("""\ +CREATE OR REPLACE FUNCTION process_user_audit() RETURNS TRIGGER AS $user_audit$ + BEGIN + IF (TG_OP = 'DELETE') THEN + INSERT INTO user_audit_logs (timestamp, operation, old_values) + SELECT now(), 'DELETE', row_to_json(OLD); + RETURN OLD; + ELSIF (TG_OP = 'UPDATE') THEN + INSERT INTO user_audit_logs (timestamp, operation, old_values, new_values) + SELECT now(), 'UPDATE', row_to_json(OLD), row_to_json(NEW); + RETURN NEW; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO user_audit_logs (timestamp, operation, new_values) + SELECT now(), 'INSERT', row_to_json(NEW); + RETURN NEW; + END IF; + RETURN NULL; + END; +$user_audit$ LANGUAGE plpgsql;""") + + exist = session.scalar( + "SELECT exists (SELECT * FROM pg_trigger WHERE tgname = 'user_audit')") + session.execute(("DROP TRIGGER user_audit ON \"User\"; " if exist else "") + """\ +CREATE TRIGGER user_audit +AFTER INSERT OR UPDATE OR DELETE ON "User" + FOR EACH ROW EXECUTE PROCEDURE process_user_audit();""") + def add_foreign_key_column_if_not_exist( table_name, From 371322875c2739dcb694e85f7b4f0946d29e08c3 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 18 Sep 2019 14:36:26 -0500 Subject: [PATCH 07/15] Synapse URL --- fence/resources/openid/synapse_oauth2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fence/resources/openid/synapse_oauth2.py b/fence/resources/openid/synapse_oauth2.py index 7c57ed59f1..9952a90635 100644 --- a/fence/resources/openid/synapse_oauth2.py +++ b/fence/resources/openid/synapse_oauth2.py @@ -10,7 +10,8 @@ class SynapseOauth2Client(Oauth2ClientBase): """ - ISSUER = "https://www.synapse.org/auth/v1" + ISSUER = "https://repo-prod.prod.sagebase.org/auth/v1/" + ISSUER = "https://repo-staging.prod.sagebase.org/auth/v1/" REQUIRED_CLAIMS = {"give_name", "family_name", "email"} OPTIONAL_CLAIMS = {"company"} SYSTEM_CLAIMS = {"sub", "exp"} From b0b0f0d02afe860a1d9cd02998b130c15aebb751 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 20 Sep 2019 10:25:34 -0500 Subject: [PATCH 08/15] fix Synapse according to staging.synapse.org --- fence/config-default.yaml | 2 + fence/resources/openid/synapse_oauth2.py | 69 ++++++++++++++++++++---- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 4b211b9652..25f832ba86 100644 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -652,4 +652,6 @@ ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS: # seconds if the team matches. DREAM_CHALLENGE_TEAM: 'DREAM' DREAM_CHALLENGE_GROUP: 'DREAM' +SYNAPSE_URI: 'https://repo-prod.prod.sagebase.org/auth/v1/' +SYNAPSE_DISCOVERY_URL: SYNAPSE_AUTHZ_TTL: 86400 \ No newline at end of file diff --git a/fence/resources/openid/synapse_oauth2.py b/fence/resources/openid/synapse_oauth2.py index 9952a90635..d968577555 100644 --- a/fence/resources/openid/synapse_oauth2.py +++ b/fence/resources/openid/synapse_oauth2.py @@ -1,6 +1,11 @@ import json +import jwt +from jwt.algorithms import RSAAlgorithm +from jwt.utils import to_base64url_uint + from .idp_oauth2 import Oauth2ClientBase +from ...config import config class SynapseOauth2Client(Oauth2ClientBase): @@ -10,10 +15,21 @@ class SynapseOauth2Client(Oauth2ClientBase): """ - ISSUER = "https://repo-prod.prod.sagebase.org/auth/v1/" - ISSUER = "https://repo-staging.prod.sagebase.org/auth/v1/" - REQUIRED_CLAIMS = {"give_name", "family_name", "email"} - OPTIONAL_CLAIMS = {"company"} + REQUIRED_CLAIMS = {"given_name", "family_name", "email", "email_verified"} + OPTIONAL_CLAIMS = { + "company", + "userid", + "orcid", + "is_certified", + "is_validated", + "validated_given_name", + "validated_family_name", + "validated_location", + "validated_email", + "validated_company", + "validated_orcid", + "validated_at", + } SYSTEM_CLAIMS = {"sub", "exp"} CUSTOM_CLAIMS = {"team"} @@ -22,7 +38,10 @@ def __init__(self, settings, logger, HTTP_PROXY=None): settings, logger, scope="openid", - discovery_url=self.ISSUER + "/.well-known/openid-configuration", + # The default discovery URL on Synapse staging is not serving the correct + # info. Providing a workaround here for overwriting. + discovery_url=config["SYNAPSE_DISCOVERY_URL"] + or (config["SYNAPSE_URI"] + "/.well-known/openid-configuration"), idp="Synapse", HTTP_PROXY=HTTP_PROXY, ) @@ -32,12 +51,12 @@ def get_auth_url(self): Get authorization uri from discovery doc """ authorization_endpoint = self.get_value_from_discovery_doc( - "authorization_endpoint", self.ISSUER + "/oauth2/authorize" + "authorization_endpoint", config["SYNAPSE_URI"] + "/oauth2/authorize" ) claims = dict( id_token=dict( - team=dict(values=["1"]), + team=dict(values=[config["DREAM_CHALLENGE_TEAM"]]), **{ claim: dict(essential=claim in self.REQUIRED_CLAIMS) for claim in self.REQUIRED_CLAIMS | self.OPTIONAL_CLAIMS @@ -52,15 +71,42 @@ def get_auth_url(self): return uri + def load_key(self, jwks_endpoint): + """A custom method to load a Synapse "RS256" key. + + Synapse is not providing standard JWK keys: + * kty is RS256 not RSA + * e and n are not base64-encoded + """ + for key in self.get_jwt_keys(jwks_endpoint): + if key["kty"] == "RS256": + key["kty"] = "RSA" + for field in ["e", "n"]: + if key[field].isdigit(): + key[field] = to_base64url_uint(int(key[field])).decode() + return "RS256", RSAAlgorithm.from_jwk(json.dumps(key)) + + return None, None + def get_user_id(self, code): try: token_endpoint = self.get_value_from_discovery_doc( - "token_endpoint", self.ISSUER + "/oauth2/token" + "token_endpoint", config["SYNAPSE_URI"] + "/oauth2/token" ) jwks_endpoint = self.get_value_from_discovery_doc( - "jwks_uri", self.ISSUER + "/oauth2/jwks" + "jwks_uri", config["SYNAPSE_URI"] + "/oauth2/jwks" + ) + token = self.get_token(token_endpoint, code) + algorithm, key = self.load_key(jwks_endpoint) + if not key: + return dict(error="Cannot load JWK keys") + + claims = jwt.decode( + token["id_token"], + key, + options={"verify_aud": False, "verify_at_hash": False}, + algorithms=[algorithm], ) - claims = self.get_jwt_claims_identity(token_endpoint, jwks_endpoint, code) if not claims["email_verified"]: return dict(error="Email is not verified") @@ -74,7 +120,8 @@ def get_user_id(self, code): ): if claim not in self.OPTIONAL_CLAIMS and claim not in claims: return dict(error="Required claim {} not found".format(claim)) - rv[claim] = claims[claim] + if claim in claims: + rv[claim] = claims[claim] return rv except Exception as e: self.logger.exception("Can't get user info") From a61bf3863d090b416d7baf526e3f4113d1c3a1ac Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 20 Sep 2019 11:12:49 -0500 Subject: [PATCH 09/15] use userdatamodel branch for testing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 295a5d8ced..77235309f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ setuptools==36.6.0 six==1.11.0 SQLAlchemy==1.3.3 temps==0.3.0 -userdatamodel==2.0.1 +git+https://github.com/uc-cdis/userdatamodel.git@feat/synapse#egg=userdatamodel Werkzeug==0.16.0 pyyaml==5.1 retry==0.9.2 From c90e46550e44c37aa2b06d2b51fc72eb68a8f26f Mon Sep 17 00:00:00 2001 From: Fantix King Date: Mon, 23 Sep 2019 09:58:46 -0500 Subject: [PATCH 10/15] add desc and secondary to IdP buttons --- fence/blueprints/login/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/login/__init__.py b/fence/blueprints/login/__init__.py index 316fa0ea5f..382a3aa410 100644 --- a/fence/blueprints/login/__init__.py +++ b/fence/blueprints/login/__init__.py @@ -77,11 +77,19 @@ def absolute_login_url(provider_id): def provider_info(idp_id): if not idp_id: - return {"id": None, "name": None, "url": None} + return { + "id": None, + "name": None, + "url": None, + "desc": None, + "secondary": False, + } return { "id": idp_id, "name": idps[idp_id]["name"], "url": absolute_login_url(idp_id), + "desc": idps[idp_id].get("desc", None), + "secondary": idps[idp_id].get("secondary", False), } try: From 3f30b39b91e040ce6a8eacdad20036821f66f2b7 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Mon, 23 Sep 2019 14:58:15 -0500 Subject: [PATCH 11/15] fix typo --- fence/blueprints/login/synapse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/login/synapse.py b/fence/blueprints/login/synapse.py index eab89dd2b3..a9685e4fa1 100644 --- a/fence/blueprints/login/synapse.py +++ b/fence/blueprints/login/synapse.py @@ -31,7 +31,7 @@ def post_login(self, user, token_result): current_session.add(user) current_session.commit() - with flask.current_app.arborist.context(auth_provider="synapse"): + with flask.current_app.arborist.context(authz_provider="synapse"): if config["DREAM_CHALLENGE_TEAM"] in token_result.get("team", []): flask.current_app.arborist.add_user_to_group( user.username, From 891f780e0f739115acd5cb68a5acef2595ae2ea5 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 25 Sep 2019 14:56:15 -0500 Subject: [PATCH 12/15] CRF --- fence/resources/openid/synapse_oauth2.py | 11 +++++++---- fence/sync/sync_users.py | 2 -- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fence/resources/openid/synapse_oauth2.py b/fence/resources/openid/synapse_oauth2.py index d968577555..980928c38b 100644 --- a/fence/resources/openid/synapse_oauth2.py +++ b/fence/resources/openid/synapse_oauth2.py @@ -112,16 +112,19 @@ def get_user_id(self, code): return dict(error="Email is not verified") rv = {} + none = object() for claim in ( self.REQUIRED_CLAIMS | self.OPTIONAL_CLAIMS | self.SYSTEM_CLAIMS | self.CUSTOM_CLAIMS ): - if claim not in self.OPTIONAL_CLAIMS and claim not in claims: - return dict(error="Required claim {} not found".format(claim)) - if claim in claims: - rv[claim] = claims[claim] + value = claims.get(claim, none) + if value is none: + if claim not in self.OPTIONAL_CLAIMS: + return dict(error="Required claim {} not found".format(claim)) + else: + rv[claim] = value return rv except Exception as e: self.logger.exception("Can't get user info") diff --git a/fence/sync/sync_users.py b/fence/sync/sync_users.py index 10f0a357f0..5bc7f9eb13 100644 --- a/fence/sync/sync_users.py +++ b/fence/sync/sync_users.py @@ -1078,8 +1078,6 @@ def _update_arborist(self, session, user_yaml): policies = user_yaml.authz.get("policies", []) for policy in policies: policy_id = policy.pop("id") - if not policy_id: - continue try: response = self.arborist_client.update_policy( policy_id, policy, create_if_not_exist=True From dcd9193808f09684d46e6cb32bbf1916c1f3ef5c Mon Sep 17 00:00:00 2001 From: Fantix King Date: Wed, 25 Sep 2019 17:19:48 -0500 Subject: [PATCH 13/15] black --- fence/__init__.py | 6 ++---- fence/models.py | 23 ++++++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/fence/__init__.py b/fence/__init__.py index ea421643e1..b6ce220df6 100644 --- a/fence/__init__.py +++ b/fence/__init__.py @@ -274,11 +274,9 @@ def _setup_oidc_clients(app): ) # Add OIDC client for Synapse if configured. - if 'synapse' in oidc: + if "synapse" in oidc: app.synapse_client = SynapseClient( - oidc["synapse"], - HTTP_PROXY=config.get("HTTP_PROXY"), - logger=logger, + oidc["synapse"], HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger ) # Add OIDC client for Microsoft if configured. diff --git a/fence/models.py b/fence/models.py index 3e586cbda5..a31d112746 100644 --- a/fence/models.py +++ b/fence/models.py @@ -24,7 +24,8 @@ Text, MetaData, Table, - text) + text, +) from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.orm import relationship, backref from sqlalchemy.sql import func @@ -691,15 +692,14 @@ def migrate(driver): add_column_if_not_exist( table_name=User.__tablename__, - column=Column( - "additional_info", JSONB(), server_default=text("'{}'") - ), + column=Column("additional_info", JSONB(), server_default=text("'{}'")), driver=driver, metadata=md, ) with driver.session as session: - session.execute("""\ + session.execute( + """\ CREATE OR REPLACE FUNCTION process_user_audit() RETURNS TRIGGER AS $user_audit$ BEGIN IF (TG_OP = 'DELETE') THEN @@ -717,14 +717,19 @@ def migrate(driver): END IF; RETURN NULL; END; -$user_audit$ LANGUAGE plpgsql;""") +$user_audit$ LANGUAGE plpgsql;""" + ) exist = session.scalar( - "SELECT exists (SELECT * FROM pg_trigger WHERE tgname = 'user_audit')") - session.execute(("DROP TRIGGER user_audit ON \"User\"; " if exist else "") + """\ + "SELECT exists (SELECT * FROM pg_trigger WHERE tgname = 'user_audit')" + ) + session.execute( + ('DROP TRIGGER user_audit ON "User"; ' if exist else "") + + """\ CREATE TRIGGER user_audit AFTER INSERT OR UPDATE OR DELETE ON "User" - FOR EACH ROW EXECUTE PROCEDURE process_user_audit();""") + FOR EACH ROW EXECUTE PROCEDURE process_user_audit();""" + ) def add_foreign_key_column_if_not_exist( From 12dd49653e7b4e89604ac2e97b462221fb7c09bc Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 27 Sep 2019 14:20:15 -0500 Subject: [PATCH 14/15] bump userdatamodel --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77235309f4..7762f8b151 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ setuptools==36.6.0 six==1.11.0 SQLAlchemy==1.3.3 temps==0.3.0 -git+https://github.com/uc-cdis/userdatamodel.git@feat/synapse#egg=userdatamodel +userdatamodel==2.1.1 Werkzeug==0.16.0 pyyaml==5.1 retry==0.9.2 From bc10e396c46affdcbad0a6b7cf47ce441e684d8b Mon Sep 17 00:00:00 2001 From: Fantix King Date: Fri, 27 Sep 2019 14:43:15 -0500 Subject: [PATCH 15/15] fix trigger for json/jsonb --- fence/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fence/models.py b/fence/models.py index a31d112746..2a9af0bc92 100644 --- a/fence/models.py +++ b/fence/models.py @@ -704,15 +704,15 @@ def migrate(driver): BEGIN IF (TG_OP = 'DELETE') THEN INSERT INTO user_audit_logs (timestamp, operation, old_values) - SELECT now(), 'DELETE', row_to_json(OLD); + SELECT now(), 'DELETE', to_jsonb(OLD); RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN INSERT INTO user_audit_logs (timestamp, operation, old_values, new_values) - SELECT now(), 'UPDATE', row_to_json(OLD), row_to_json(NEW); + SELECT now(), 'UPDATE', to_jsonb(OLD), to_jsonb(NEW); RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO user_audit_logs (timestamp, operation, new_values) - SELECT now(), 'INSERT', row_to_json(NEW); + SELECT now(), 'INSERT', to_jsonb(NEW); RETURN NEW; END IF; RETURN NULL;