Merge branch 'integration202210' of https://github.com/uc-cdis/fence …

…into stable

.github/workflows/buildpipeline.yaml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+    push:
+    pull_request:
+      types: [opened, reopened]
+
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+
+jobs:
+  Security:
+    name: Security Pipeline
+    uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
+    secrets: inherit
+
+  ci:
+    name: Build Image and Push
+    # TODO Add this line back once we update to Python 3.9 from 3.6
+    # needs: Security
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
     -   id: black
 -   repo: git@github.com:Yelp/detect-secrets

.secrets.baseline

@@ -189,7 +189,7 @@
         "filename": "fence/utils.py",
         "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
         "is_verified": false,
-        "line_number": 105
+        "line_number": 110
       }
     ],
     "migrations/versions/e4c7b0ab68d3_create_tables.py": [
@@ -201,20 +201,36 @@
         "line_number": 22
       }
     ],
+    "migrations/versions/ea7e1b843f82_optional_client_redirect_uri.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "migrations/versions/ea7e1b843f82_optional_client_redirect_uri.py",
+        "hashed_secret": "bb2372fb50034d559d2920e7229fb5879cf1be72",
+        "is_verified": false,
+        "line_number": 13
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "migrations/versions/ea7e1b843f82_optional_client_redirect_uri.py",
+        "hashed_secret": "adb1fcd33b07abf9b6a064745759accea5cb341f",
+        "is_verified": false,
+        "line_number": 14
+      }
+    ],
     "tests/conftest.py": [
       {
         "type": "Private Key",
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1516
+        "line_number": 1574
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1539
+        "line_number": 1597
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -238,6 +254,13 @@
         "hashed_secret": "768b7fe00de4fd233c0c72375d12f87ce9670144",
         "is_verified": false,
         "line_number": 476
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/credentials/google/test_credentials.py",
+        "hashed_secret": "22afbfecd4124e2eb0e2a79fafdf62b207a8f8c7",
+        "is_verified": false,
+        "line_number": 580
       }
     ],
     "tests/data/test_indexed_file.py": [
@@ -267,6 +290,22 @@
         "line_number": 48
       }
     ],
+    "tests/migrations/test_ea7e1b843f82.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/migrations/test_ea7e1b843f82.py",
+        "hashed_secret": "adb1fcd33b07abf9b6a064745759accea5cb341f",
+        "is_verified": false,
+        "line_number": 27
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/migrations/test_ea7e1b843f82.py",
+        "hashed_secret": "bb2372fb50034d559d2920e7229fb5879cf1be72",
+        "is_verified": false,
+        "line_number": 44
+      }
+    ],
     "tests/ras/test_ras.py": [
       {
         "type": "Hex High Entropy String",
@@ -276,6 +315,15 @@
         "line_number": 120
       }
     ],
+    "tests/scripting/test_fence-create.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/scripting/test_fence-create.py",
+        "hashed_secret": "e5e9fa1ba31ecd1ae84f75caaa474f3a663f05f4",
+        "is_verified": false,
+        "line_number": 221
+      }
+    ],
     "tests/test-fence-config.yaml": [
       {
         "type": "Basic Auth Credentials",
@@ -286,5 +334,5 @@
       }
     ]
   },
-  "generated_at": "2022-07-08T21:32:25Z"
+  "generated_at": "2022-08-12T15:25:33Z"
 }

.travis.yml

@@ -11,8 +11,7 @@ addons:
   postgresql: "9.6"
 
 install:
-  - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
-  - source $HOME/.poetry/env
+  - pip install poetry
   - poetry install -vv
   - psql -c 'SELECT version();' -U postgres
   - psql -U postgres -c "create database fence_test_tmp"

Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update \
     && apt-get -y install vim \
     libmcrypt4 libmhash2 mcrypt \
     && apt-get clean \
-    && rm -rf /var/lib/apt/lists/  
+    && rm -rf /var/lib/apt/lists/
 
 RUN mkdir -p /var/www/$appname \
     && mkdir -p /var/www/.cache/Python-Eggs/ \

README.md

@@ -415,7 +415,7 @@ We use JSON Web Tokens (JWTs) as the format for all tokens of the following type
       }
     }
   },
-  "iss": "https://bionimbus-pdc.opensciencedatacloud.org",
+  "iss": "https://commons.org",
   "jti": "3ae2910b-0294-43dc-af2a-03fd60082aef",
   "exp": 1516983302,
   "iat": 1516982102,
@@ -454,7 +454,7 @@ We use JSON Web Tokens (JWTs) as the format for all tokens of the following type
       }
     }
   },
-  "iss": "https://bionimbus-pdc.opensciencedatacloud.org",
+  "iss": "https://commons.org",
   "jti": "2e6ade06-5afb-4ce7-9ab5-e206225ce291",
   "exp": 1516983302,
   "iat": 1516982102
@@ -473,7 +473,7 @@ We use JSON Web Tokens (JWTs) as the format for all tokens of the following type
     "user",
     "test-client"
   ],
-  "iss": "https://bionimbus-pdc.opensciencedatacloud.org",
+  "iss": "https://commons.org",
   "jti": "c72e5573-39fa-4391-a445-191e370b7cc5",
   "exp": 1517010902,
   "iat": 1516982102
@@ -515,6 +515,28 @@ To specify allowed scopes, use the `allowed-scopes` argument:
 fence-create client-create ...  --allowed-scopes openid user data
 ```
 
+#### Register an Oauth Client for a Client Credentials flow
+
+The OAuth2 Client Credentials flow is used for machine-to-machine communication and scenarios in which typical authentication schemes like username + password do not make sense. The system authenticates and authorizes the app rather than a user. See the [OAuth2 specification](https://www.rfc-editor.org/rfc/rfc6749#section-4.4) for more details.
+
+As a Gen3 commons administrator, if you want to create an OAuth client for a client credentials flow:
+
+```bash
+fence-create client-create --client CLIENT_NAME --grant-types client_credentials
+```
+
+This command will return a client ID and client secret, which you can then use to obtain an access token:
+
+```bash
+curl --request POST https://FENCE_URL/oauth2/token?grant_type=client_credentials -d scope="openid user" --user CLIENT_ID:CLIENT_SECRET
+```
+
+NOTE: In Gen3, you can grant specific access to a client the same way you would to a user. See the [user.yaml guide](https://github.com/uc-cdis/fence/blob/master/docs/user.yaml_guide.md) for more details.
+
+NOTE: Client credentials tokens are not linked to a user. They are not supported by all Gen3 endpoints.
+
+NOTE: The recommendation is to rotate these credentials at least once a year. Credentials expiration is not enforced at the moment but may be in the future.
+
 #### Modify OAuth Client
 
 ```bash

bin/fence_create.py

@@ -69,11 +69,10 @@ def parse_arguments():
 
     client_create = subparsers.add_parser("client-create")
     client_create.add_argument("--client", required=True)
-    client_create.add_argument("--urls", required=True, nargs="+")
+    client_create.add_argument("--urls", nargs="+")
     client_create.add_argument(
         "--username",
         help="user(can represent an organization) that owns the client",
-        required=True,
     )
     client_create.add_argument(
         "--external",
@@ -89,7 +88,7 @@ def parse_arguments():
     )
     client_create.add_argument(
         "--grant-types",
-        help="which OAuth2 grant types are enabled for this client",
+        help="which OAuth2 grant types are enabled for this client (default: authorization_code and refresh_token)",
         nargs="+",
     )
     client_create.add_argument(

bin/old_migration_script.py

@@ -4,7 +4,7 @@
 pre-Alembic version to a post-Alembic version.
 
 DO NOT ADD NEW MIGRATIONS TO THIS SCRIPT.
-Create a new Alembic version instead.
+Create a new Alembic revision instead.
 """
 
 

fence/auth.py

@@ -234,12 +234,14 @@ def has_oauth(scope=None):
         )
     except JWTError as e:
         raise Unauthorized("failed to validate token: {}".format(e))
-    user_id = access_token_claims["sub"]
-    user = current_session.query(User).filter_by(id=int(user_id)).first()
-    if not user:
-        raise Unauthorized("no user found with id: {}".format(user_id))
-    # set some application context for current user and client id
-    flask.g.user = user
+    if "sub" in access_token_claims:
+        user_id = access_token_claims["sub"]
+        user = current_session.query(User).filter_by(id=int(user_id)).first()
+        if not user:
+            raise Unauthorized("no user found with id: {}".format(user_id))
+        # set some application context for current user
+        flask.g.user = user
+    # set some application context for current client id
     # client_id should be None if the field doesn't exist or is empty
     flask.g.client_id = access_token_claims.get("azp") or None
     flask.g.token = access_token_claims