Merge branch 'integration202106' of https://github.com/uc-cdis/fence …

…into stable

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "poetry.lock",
     "lines": null
   },
-  "generated_at": "2021-04-01T15:04:02Z",
+  "generated_at": "2021-04-12T21:59:04Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -122,31 +122,31 @@
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1903,
+        "line_number": 1914,
         "type": "Private Key"
       },
       {
         "hashed_secret": "bb8e48bd1e73662027a0f0b876b695d4c18f5ed4",
         "is_verified": false,
-        "line_number": 1903,
+        "line_number": 1914,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7861ab65194de92776ab9cd06d4d7e7e1ec2c36d",
         "is_verified": false,
-        "line_number": 1983,
+        "line_number": 1994,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d6b66ddd9ea7dbe760114bfe9a97352a5e139134",
         "is_verified": false,
-        "line_number": 2005,
+        "line_number": 2016,
         "type": "JSON Web Token"
       },
       {
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
         "is_verified": false,
-        "line_number": 2017,
+        "line_number": 2028,
         "type": "Base64 High Entropy String"
       }
     ],

README.md

@@ -99,6 +99,7 @@ At the moment, supported IDPs include:
   - NIH iTrust
   - InCommon
   - eduGAIN
+- CILogon
 - Cognito
 - Synapse
 - Microsoft

docs/user.yaml_guide.md

@@ -8,6 +8,9 @@
   - [Notes](#notes)
 - [Deprecated format](#deprecated-format)
   - [For Gen3 Data Commons that do not use Arborist or use the Google Data Access method of Google Service Account Registration](#for-gen3-data-commons-that-do-not-use-arborist-or-use-the-google-data-access-method-of-google-service-account-registration)
+- [Public data](#public-data)
+  - [Public access to resources](#public-access-to-resources)
+  - [The "/open" resource](#the-open-resource)
 
 ## Introduction
 
@@ -236,3 +239,71 @@ users:
 ```
 
 The `user_project_to_resource` section can be used to avoid specifying a resource path for each `users.projects.resource`.
+
+## Public data
+
+> What is involved in making a project "public"; that is, making both the metadata and object files accessible to anyone who visits the Data Commons?
+
+### Public access to resources
+
+Arborist can be configured to apply a policy to _all_ users who visit the system. This is done via the special `user.yaml` field `anonymous_policies`. Note that the same can be done with `all_users_policies` instead of `anonymous_policies` if access should be granted to all authenticated users instead of both authenticated and non-authenticated users.
+
+The example below shows the setup for a program `PUBLIC_PROGRAM` and a project `PROJECT_1` under it. Because the policy `PUBLIC_PROGRAM_reader`, which grants access to this program, is in `anonymous_policies`, this program and all the subresources under it are accessible to all users.
+
+Structured graph data in program `PUBLIC_PROGRAM` and data files whose indexd records' `authz` field includes `/programs/PUBLIC_PROGRAM/projects/PROJECT_1` will both be publicly accessible.
+
+```
+authz:
+    # policies automatically given to anyone, even if they haven't authenticated
+    anonymous_policies:
+    - PUBLIC_PROGRAM_reader
+
+    resources:
+    - name: programs
+      subresources:
+      - name: PUBLIC_PROGRAM
+        subresources:
+        - name: projects
+          subresources:
+          - name: PROJECT_1
+
+    policies:
+    - id: PUBLIC_PROGRAM_reader
+      role_ids:
+      - reader
+      - storage_reader
+      resource_paths:
+      - /programs/PUBLIC_PROGRAM
+```
+
+Arborist is very flexible: we could define an open policy per public program, or per public project, or even a single open policy with a list of all open resources.
+
+### The "/open" resource
+
+> Note that we may alter the behavior around "/open" in the future so as not to have hard-coded resource logic in Fence, so relying on this behavior is not recommended.
+
+`/open` is a special resource supported by Gen3. It is only used for data files (in the `authz` field of indexd records).
+
+An indexd record's `authz` field containing the resouce path `/open` means that Fence doesn't need to sign presigned URLs. Fence will assume the bucket is public. When a user tries to download the file, Fence will return a non-signed URL.
+
+If the bucket is _not_ public but the data should be publicly accessible, public access should be granted via the `user.yaml` file but `/open` should _not_ be added in the `authz` field.
+
+The example below shows how to set up public access to the `/open` resource.
+
+```
+authz:
+    # policies automatically given to anyone, even if they haven't authenticated
+    anonymous_policies:
+    - open_data_reader
+
+    resources:
+    - name: open
+
+    policies:
+    - id: open_data_reader
+      role_ids:
+      - reader
+      - storage_reader
+      resource_paths:
+      - /open
+```

fence/__init__.py

@@ -18,11 +18,13 @@
 from fence.oidc.server import server
 from fence.resources.audit_service_client import AuditServiceClient
 from fence.resources.aws.boto_manager import BotoManager
+from fence.resources.openid.cilogon_oauth2 import CilogonOauth2Client as CilogonClient
 from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client as CognitoClient
 from fence.resources.openid.google_oauth2 import GoogleOauth2Client as GoogleClient
 from fence.resources.openid.microsoft_oauth2 import (
     MicrosoftOauth2Client as MicrosoftClient,
 )
+from fence.resources.openid.okta_oauth2 import OktaOauth2Client as OktaClient
 from fence.resources.openid.orcid_oauth2 import OrcidOauth2Client as ORCIDClient
 from fence.resources.openid.synapse_oauth2 import SynapseOauth2Client as SynapseClient
 from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient
@@ -358,12 +360,28 @@ def _setup_oidc_clients(app):
             logger=logger,
         )
 
+    # Add OIDC client for Okta if configured
+    if "okta" in oidc:
+        app.okta_client = OktaClient(
+            config["OPENID_CONNECT"]["okta"],
+            HTTP_PROXY=config.get("HTTP_PROXY"),
+            logger=logger,
+        )
+
     # Add OIDC client for Amazon Cognito if configured.
     if "cognito" in oidc:
         app.cognito_client = CognitoClient(
             oidc["cognito"], HTTP_PROXY=config.get("HTTP_PROXY"), logger=logger
         )
 
+    # Add OIDC client for CILogon if configured.
+    if "cilogon" in oidc:
+        app.cilogon_client = CilogonClient(
+            config["OPENID_CONNECT"]["cilogon"],
+            HTTP_PROXY=config.get("HTTP_PROXY"),
+            logger=logger,
+        )
+
     # Add OIDC client for multi-tenant fence if configured.
     if "fence" in oidc:
         app.fence_client = OAuthClient(**config["OPENID_CONNECT"]["fence"])

fence/blueprints/login/__init__.py

@@ -12,11 +12,13 @@
 
 from cdislogging import get_logger
 
+from fence.blueprints.login.cilogon import CilogonLogin, CilogonCallback
 from fence.blueprints.login.cognito import CognitoLogin, CognitoCallback
 from fence.blueprints.login.fence_login import FenceLogin, FenceCallback
 from fence.blueprints.login.google import GoogleLogin, GoogleCallback
 from fence.blueprints.login.shib import ShibbolethLogin, ShibbolethCallback
 from fence.blueprints.login.microsoft import MicrosoftLogin, MicrosoftCallback
+from fence.blueprints.login.okta import OktaLogin, OktaCallback
 from fence.blueprints.login.orcid import ORCIDLogin, ORCIDCallback
 from fence.blueprints.login.ras import RASLogin, RASCallback
 from fence.blueprints.login.synapse import SynapseLogin, SynapseCallback
@@ -34,8 +36,10 @@
     "orcid": "orcid",
     "synapse": "synapse",
     "microsoft": "microsoft",
+    "okta": "okta",
     "cognito": "cognito",
     "ras": "ras",
+    "cilogon": "cilogon",
 }
 
 
@@ -275,6 +279,10 @@ def provider_info(login_details):
             MicrosoftCallback, "/microsoft/login", strict_slashes=False
         )
 
+    if "okta" in configured_idps:
+        blueprint_api.add_resource(OktaLogin, "/okta", strict_slashes=False)
+        blueprint_api.add_resource(OktaCallback, "/okta/login", strict_slashes=False)
+
     if "cognito" in configured_idps:
         blueprint_api.add_resource(CognitoLogin, "/cognito", strict_slashes=False)
         blueprint_api.add_resource(
@@ -286,6 +294,13 @@ def provider_info(login_details):
         blueprint_api.add_resource(
             ShibbolethCallback, "/shib/login", strict_slashes=False
         )
+
+    if "cilogon" in configured_idps:
+        blueprint_api.add_resource(CilogonLogin, "/cilogon", strict_slashes=False)
+        blueprint_api.add_resource(
+            CilogonCallback, "/cilogon/login", strict_slashes=False
+        )
+
     return blueprint
 
 

fence/blueprints/login/cilogon.py

@@ -0,0 +1,21 @@
+import flask
+
+from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback
+
+CILOGON_IDP_NAME = "cilogon"
+
+
+class CilogonLogin(DefaultOAuth2Login):
+    def __init__(self):
+        super(CilogonLogin, self).__init__(
+            idp_name=CILOGON_IDP_NAME, client=flask.current_app.cilogon_client
+        )
+
+
+class CilogonCallback(DefaultOAuth2Callback):
+    def __init__(self):
+        super(CilogonCallback, self).__init__(
+            idp_name=CILOGON_IDP_NAME,
+            client=flask.current_app.cilogon_client,
+            username_field="sub",
+        )

fence/blueprints/login/okta.py

@@ -0,0 +1,20 @@
+import flask
+
+from fence.resources.openid.okta_oauth2 import OKTA_IDP_NAME
+from fence.blueprints.login.base import DefaultOAuth2Login, DefaultOAuth2Callback
+
+
+class OktaLogin(DefaultOAuth2Login):
+    def __init__(self):
+        super(OktaLogin, self).__init__(
+            idp_name=OKTA_IDP_NAME,
+            client=flask.current_app.okta_client,
+        )
+
+
+class OktaCallback(DefaultOAuth2Callback):
+    def __init__(self):
+        super(OktaCallback, self).__init__(
+            idp_name=OKTA_IDP_NAME,
+            client=flask.current_app.okta_client,
+        )

fence/config-default.yaml

@@ -174,6 +174,13 @@ OPENID_CONNECT:
     # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only)
     mock: false
     mock_default_user: 'test@example.com'
+  # For information on configuring an Okta tenant as an OIDC IdP refer to Okta documentation at:
+  # https://developer.okta.com/docs/reference/api/oidc/#2-okta-as-the-identity-platform-for-your-app-or-api  
+  okta:
+    discovery_url: ''
+    client_id: ''
+    client_secret: ''
+    redirect_url: '{{BASE_URL}}/login/okta/login/'
   cognito:
     # You must create a user pool in order to have a discovery url
     discovery_url: 'https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration'
@@ -184,6 +191,19 @@ OPENID_CONNECT:
     # and that IdP is a SAML IdP with no 'email_verified' outgoing claim, but it is safe
     # to assume all emails from this SAML IdP are in fact verified, we may set this to True
     assume_emails_verified: False
+  # CILogon subscribers can create and manage OIDC clients using COmanage Registry.
+  # Free tier users may request OIDC clients at https://cilogon.org/oauth2/register
+  cilogon:
+    client_id: ''
+    client_secret: ''
+    # When registering the Callback URLs for your CILogon OIDC client be
+    # sure to include the FULL url for this deployment, including the https:// scheme
+    # and server FQDN.
+    redirect_url: '{{BASE_URL}}/login/cilogon/login/'
+    # if mock is true, will fake a successful login response for login
+    # WARNING: DO NOT ENABLE IN PRODUCTION (for testing purposes only)
+    mock: false
+    mock_default_user: 'http://cilogon.org/serverT/users/64703'
   synapse:
     client_id: ''
     client_secret: ''
@@ -257,6 +277,8 @@ LOGIN_OPTIONS: [] # !!! remove the empty list to enable login options!
   #   idp: orcid
   # - name: 'Microsoft Login'
   #   idp: microsoft
+  # - name: 'Okta Login'
+  #   idp: okta
   # # Cognito login: You may want to edit the name to reflect Cognito's IdP,
   # # especially if Cognito is only using one IdP
   # - name: 'Login from Cognito'
@@ -270,6 +292,8 @@ LOGIN_OPTIONS: [] # !!! remove the empty list to enable login options!
   # - name: 'ORCID Login through other Fence'
   #   idp: fence
   #   fence_idp: orcid
+  # - name: 'CILogon Login'
+  #   idp: cilogon
   # - name: 'InCommon Login'
   #   idp: fence
   #   fence_idp: shibboleth

fence/resources/openid/cilogon_oauth2.py

@@ -0,0 +1,51 @@
+from .idp_oauth2 import Oauth2ClientBase
+
+
+class CilogonOauth2Client(Oauth2ClientBase):
+    """
+    client for interacting with CILogon OIDC
+    """
+
+    CILOGON_DISCOVERY_URL = "https://cilogon.org/.well-known/openid-configuration"
+
+    def __init__(self, settings, logger, HTTP_PROXY=None):
+        super(CilogonOauth2Client, self).__init__(
+            settings,
+            logger,
+            scope="openid email profile",
+            discovery_url=self.CILOGON_DISCOVERY_URL,
+            idp="CILogon",
+            HTTP_PROXY=HTTP_PROXY,
+        )
+
+    def get_auth_url(self):
+        """
+        Get authorization uri from discovery doc
+        """
+        authorization_endpoint = self.get_value_from_discovery_doc(
+            "authorization_endpoint", "https://cilogon.org/authorize"
+        )
+
+        uri, _ = self.session.create_authorization_url(
+            authorization_endpoint, prompt="login"
+        )
+
+        return uri
+
+    def get_user_id(self, code):
+        try:
+            token_endpoint = self.get_value_from_discovery_doc(
+                "token_endpoint", "https://cilogon.org/oauth2/token"
+            )
+            jwks_endpoint = self.get_value_from_discovery_doc(
+                "jwks_uri", "https://cilogon.org/oauth2/certs"
+            )
+            claims = self.get_jwt_claims_identity(token_endpoint, jwks_endpoint, code)
+
+            if claims["sub"]:
+                return {"sub": claims["sub"]}
+            else:
+                return {"error": "Can't get user's CILogon sub"}
+        except Exception as e:
+            self.logger.exception("Can't get user info")
+            return {"error": "Can't get your CILogon sub: {}".format(e)}