Merge branch 'fence_external_contributor_change1' of github.com:thema…

…rcelor/fence into fence_external_contributor_change1

.pre-commit-config.yaml

@@ -13,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: stable
     hooks:
     -   id: black

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "poetry.lock",
     "lines": null
   },
-  "generated_at": "2021-02-19T16:52:13Z",
+  "generated_at": "2021-04-12T21:59:04Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -58,14 +58,6 @@
     }
   ],
   "results": {
-    "bin/fence-create": [
-      {
-        "hashed_secret": "07cb451426a70236a0047e0f390f2bd1d79fd6d3",
-        "is_verified": false,
-        "line_number": 515,
-        "type": "Secret Keyword"
-      }
-    ],
     "fence/blueprints/storage_creds/google.py": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
@@ -92,7 +84,7 @@
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
         "is_verified": false,
-        "line_number": 511,
+        "line_number": 516,
         "type": "Secret Keyword"
       }
     ],
@@ -130,45 +122,45 @@
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1863,
+        "line_number": 1914,
         "type": "Private Key"
       },
       {
         "hashed_secret": "bb8e48bd1e73662027a0f0b876b695d4c18f5ed4",
         "is_verified": false,
-        "line_number": 1863,
+        "line_number": 1914,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7861ab65194de92776ab9cd06d4d7e7e1ec2c36d",
         "is_verified": false,
-        "line_number": 1943,
+        "line_number": 1994,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d6b66ddd9ea7dbe760114bfe9a97352a5e139134",
         "is_verified": false,
-        "line_number": 1965,
+        "line_number": 2016,
         "type": "JSON Web Token"
       },
       {
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
         "is_verified": false,
-        "line_number": 1977,
+        "line_number": 2028,
         "type": "Base64 High Entropy String"
       }
     ],
     "tests/conftest.py": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1037,
+        "line_number": 1123,
         "type": "Private Key"
       },
       {
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1060,
+        "line_number": 1146,
         "type": "Base64 High Entropy String"
       }
     ],
@@ -192,7 +184,7 @@
       {
         "hashed_secret": "d300421e208bfd0d432294de15169fd9b8975def",
         "is_verified": false,
-        "line_number": 40,
+        "line_number": 41,
         "type": "Secret Keyword"
       }
     ],

Dockerfile

@@ -1,15 +1,16 @@
 # To run: docker run --rm -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
 # To check running container: docker exec -it fence /bin/bash
 
-FROM quay.io/cdis/python-nginx:pybase3-1.4.2
+FROM quay.io/cdis/python-nginx:pybase3-1.5.0
 
 ENV appname=fence
 
-RUN apk update \
-    && apk add postgresql-libs postgresql-dev libffi-dev libressl-dev \
-    && apk add linux-headers musl-dev gcc \
-    && apk add curl bash git vim make lftp \
-    && apk update && apk add openssh && apk add libmcrypt-dev
+RUN pip install --upgrade pip
+RUN apk add --update \
+    postgresql-libs postgresql-dev libffi-dev libressl-dev \
+    linux-headers musl-dev gcc g++ \
+    curl bash git vim make lftp \
+    openssh libmcrypt-dev
 
 RUN mkdir -p /var/www/$appname \
     && mkdir -p /var/www/.cache/Python-Eggs/ \

README.md

@@ -185,7 +185,7 @@ Additionally, see the [OAuth2 specification](https://tools.ietf.org/html/rfc6749
 Currently fence works with another Gen3 service named
 [arborist](https://github.com/uc-cdis/arborist) to implement attribute-based access
 control for commons users. The YAML file of access control information (see
-[#create-user-access-file]()) contains a section `authz` which are data sent to
+[#create-user-access-file](#create-user-access-file)) contains a section `authz` which are data sent to
 arborist in order to set up the access control model.
 
 ## Accessing Data

fence/__init__.py

@@ -5,6 +5,7 @@
 import flask
 from flask_cors import CORS
 from flask_sqlalchemy_session import flask_scoped_session, current_session
+from urllib.parse import urljoin
 from userdatamodel.driver import SQLAlchemyDriver
 
 from fence.auth import logout, build_redirect_url
@@ -15,6 +16,7 @@
 from fence.models import migrate
 from fence.oidc.client import query_client
 from fence.oidc.server import server
+from fence.resources.audit_service_client import AuditServiceClient
 from fence.resources.aws.boto_manager import BotoManager
 from fence.resources.openid.cognito_oauth2 import CognitoOauth2Client as CognitoClient
 from fence.resources.openid.google_oauth2 import GoogleOauth2Client as GoogleClient
@@ -261,6 +263,7 @@ def app_config(
     app.config.update(**config._configs)
 
     _setup_arborist_client(app)
+    _setup_audit_service_client(app)
     _setup_data_endpoint_and_boto(app)
     _load_keys(app, root_dir)
     _set_authlib_cfgs(app)
@@ -315,13 +318,6 @@ def _set_authlib_cfgs(app):
 
 
 def _setup_oidc_clients(app):
-    if config["LOGIN_OPTIONS"]:
-        enabled_idp_ids = [option["idp"] for option in config["LOGIN_OPTIONS"]]
-    else:
-        # fall back on "providers"
-        enabled_idp_ids = list(
-            config.get("ENABLED_IDENTITY_PROVIDERS", {}).get("providers", {}).keys()
-        )
     oidc = config.get("OPENID_CONNECT", {})
 
     # Add OIDC client for Google if configured.
@@ -369,8 +365,7 @@ def _setup_oidc_clients(app):
         )
 
     # Add OIDC client for multi-tenant fence if configured.
-    configured_fence = "fence" in oidc and "fence" in enabled_idp_ids
-    if configured_fence:
+    if "fence" in oidc:
         app.fence_client = OAuthClient(**config["OPENID_CONNECT"]["fence"])
 
 
@@ -379,6 +374,19 @@ def _setup_arborist_client(app):
         app.arborist = ArboristClient(arborist_base_url=config["ARBORIST"])
 
 
+def _setup_audit_service_client(app):
+    # Initialize the client regardless of whether audit logs are enabled. This
+    # allows us to call `app.audit_service_client.create_x_log()` from
+    # anywhere without checking if audit logs are enabled. The client
+    # checks that for us.
+    service_url = app.config.get("AUDIT_SERVICE") or urljoin(
+        app.config["BASE_URL"], "/audit"
+    )
+    app.audit_service_client = AuditServiceClient(
+        service_url=service_url, logger=logger
+    )
+
+
 @app.errorhandler(Exception)
 def handle_error(error):
     """

fence/auth.py

@@ -25,7 +25,6 @@
 def get_jwt():
     """
     Return the user's JWT from authorization header. Requires flask application context.
-
     Raises:
         - Unauthorized, if header is missing or not in the correct format
     """
@@ -44,12 +43,10 @@ def get_jwt():
 def build_redirect_url(hostname, path):
     """
     Compute a redirect given a hostname and next path where
-
     Args:
         hostname (str): may be empty string or a bare hostname or
                a hostname with a protocal attached (https?://...)
         path (int): is a path to attach to hostname
-
     Return:
         string url suitable for flask.redirect
     """
@@ -60,36 +57,67 @@ def build_redirect_url(hostname, path):
     return redirect_base + path
 
 
-def login_user(request, username, provider):
-    user = query_for_user(session=current_session, username=username)
+def login_user(username, provider, fence_idp=None, shib_idp=None):
+    """
+    Login a user with the given username and provider. Set values in Flask
+    session to indicate the user being logged in. In addition, commit the user
+    and associated idp information to the db.
 
-    if not user:
+    Args:
+        username (str): specific username of user to be logged in
+        provider (str): specfic idp of user to be logged in
+
+    """
+
+    def set_flask_session_values(user):
+        """
+        Helper fuction to set user values in the session.
+
+        Args:
+            user (User): User object
+        """
+        flask.session["username"] = user.username
+        flask.session["user_id"] = str(user.id)
+        flask.session["provider"] = user.identity_provider.name
+        if fence_idp:
+            flask.session["fence_idp"] = fence_idp
+        if shib_idp:
+            flask.session["shib_idp"] = shib_idp
+        flask.g.user = user
+        flask.g.scopes = ["_all"]
+        flask.g.token = None
+
+    user = query_for_user(session=current_session, username=username)
+    if user:
+        #  This expression is relevant to those users who already have user and
+        #  idp info persisted to the database. We return early to avoid
+        #  unnecessarily re-saving that user and idp info.
+        if user.identity_provider and user.identity_provider.name == provider:
+            set_flask_session_values(user)
+            return
+    else:
         user = User(username=username)
-        idp = (
-            current_session.query(IdentityProvider)
-            .filter(IdentityProvider.name == provider)
-            .first()
-        )
-        if not idp:
-            idp = IdentityProvider(name=provider)
-        user.identity_provider = idp
-        current_session.add(user)
-        current_session.commit()
-    flask.session["username"] = username
-    flask.session["provider"] = provider
-    flask.session["user_id"] = str(user.id)
-    flask.g.user = user
-    flask.g.scopes = ["_all"]
-    flask.g.token = None
+
+    idp = (
+        current_session.query(IdentityProvider)
+        .filter(IdentityProvider.name == provider)
+        .first()
+    )
+    if not idp:
+        idp = IdentityProvider(name=provider)
+
+    user.identity_provider = idp
+    current_session.add(user)
+    current_session.commit()
+
+    set_flask_session_values(user)
 
 
 def logout(next_url, force_era_global_logout=False):
     """
     Return a redirect which another logout from IDP or the provided redirect.
-
     Depending on the IDP, this logout will propogate. For example, if using
     another fence as an IDP, this will hit that fence's logout endpoint.
-
     Args:
         next_url (str): Final redirect desired after logout
     """
@@ -138,9 +166,7 @@ def decorator(f):
         @wraps(f)
         def wrapper(*args, **kwargs):
             if flask.session.get("username"):
-                login_user(
-                    flask.request, flask.session["username"], flask.session["provider"]
-                )
+                login_user(flask.session["username"], flask.session["provider"])
                 return f(*args, **kwargs)
 
             eppn = None
@@ -169,7 +195,7 @@ def wrapper(*args, **kwargs):
                 username = eppn.split("!")[-1]
                 flask.session["username"] = username
                 flask.session["provider"] = IdentityProvider.itrust
-                login_user(flask.request, username, flask.session["provider"])
+                login_user(username, flask.session["provider"])
                 return f(*args, **kwargs)
             else:
                 raise Unauthorized("Please login")
@@ -179,28 +205,6 @@ def wrapper(*args, **kwargs):
     return decorator
 
 
-def handle_login(scope):
-    if flask.session.get("username"):
-        login_user(flask.request, flask.session["username"], flask.session["provider"])
-
-    eppn = flask.request.headers.get(config["SHIBBOLETH_HEADER"])
-
-    if config.get("MOCK_AUTH") is True:
-        eppn = "test"
-    # if there is authorization header for oauth
-    if "Authorization" in flask.request.headers:
-        has_oauth(scope=scope)
-    # if there is shibboleth session, then create user session and
-    # log user in
-    elif eppn:
-        username = eppn.split("!")[-1]
-        flask.session["username"] = username
-        flask.session["provider"] = IdentityProvider.itrust
-        login_user(flask.request, username, flask.session["provider"])
-    else:
-        raise Unauthorized("Please login")
-
-
 def has_oauth(scope=None):
     scope = scope or set()
     scope.update({"openid"})

fence/blueprints/data/blueprint.py

@@ -15,7 +15,6 @@
 
 logger = get_logger(__name__)
 
-
 blueprint = flask.Blueprint("data", __name__)