merge master

uc-cdis · Jun 9, 2021 · 65a607e · 65a607e
2 parents a453ce9 + c94f493
commit 65a607e
Show file tree

Hide file tree

Showing 26 changed files with 470 additions and 416 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,18 +1,18 @@
 repos:
 -   repo: git@github.com:Yelp/detect-secrets
-    rev: v0.13.1
+    rev: v1.1.0
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
         exclude: poetry.lock
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: stable
+    rev: 21.5b1
     hooks:
     -   id: black
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1,9 +1,5 @@
 {
-  "exclude": {
-    "files": "poetry.lock",
-    "lines": null
-  },
-  "generated_at": "2021-06-01T15:16:48Z",
+  "generated_at": "2021-06-09T22:00:09Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -12,8 +8,8 @@
       "name": "ArtifactoryDetector"
     },
     {
-      "base64_limit": 4.5,
-      "name": "Base64HighEntropyString"
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
     },
     {
       "name": "BasicAuthDetector"
@@ -22,8 +18,8 @@
       "name": "CloudantDetector"
     },
     {
-      "hex_limit": 3,
-      "name": "HexHighEntropyString"
+      "name": "HexHighEntropyString",
+      "limit": 3
     },
     {
       "name": "IbmCloudIamDetector"
@@ -60,154 +56,212 @@
   "results": {
     "fence/blueprints/storage_creds/google.py": [
       {
+        "type": "Private Key",
+        "filename": "fence/blueprints/storage_creds/google.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 139,
-        "type": "Private Key"
+        "line_number": 139
       }
     ],
     "fence/blueprints/storage_creds/other.py": [
       {
+        "type": "Base64 High Entropy String",
+        "filename": "fence/blueprints/storage_creds/other.py",
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
         "is_verified": false,
-        "line_number": 66,
-        "type": "Base64 High Entropy String"
+        "line_number": 66
       }
     ],
     "fence/config-default.yaml": [
       {
+        "type": "Basic Auth Credentials",
+        "filename": "fence/config-default.yaml",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 31,
-        "type": "Basic Auth Credentials"
+        "line_number": 31
       },
       {
+        "type": "Secret Keyword",
+        "filename": "fence/config-default.yaml",
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
         "is_verified": false,
-        "line_number": 554,
-        "type": "Secret Keyword"
+        "line_number": 554
       }
     ],
     "fence/local_settings.example.py": [
       {
+        "type": "Basic Auth Credentials",
+        "filename": "fence/local_settings.example.py",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Basic Auth Credentials"
+        "line_number": 6
       },
       {
+        "type": "Secret Keyword",
+        "filename": "fence/local_settings.example.py",
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
         "is_verified": false,
-        "line_number": 63,
-        "type": "Secret Keyword"
+        "line_number": 63
       }
     ],
     "fence/resources/google/utils.py": [
       {
+        "type": "Private Key",
+        "filename": "fence/resources/google/utils.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 277,
-        "type": "Private Key"
+        "line_number": 277
       }
     ],
     "fence/utils.py": [
       {
+        "type": "Secret Keyword",
+        "filename": "fence/utils.py",
         "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
         "is_verified": false,
-        "line_number": 105,
-        "type": "Secret Keyword"
+        "line_number": 105
       }
     ],
     "openapis/swagger.yaml": [
       {
+        "type": "Private Key",
+        "filename": "openapis/swagger.yaml",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1927,
-        "type": "Private Key"
+        "line_number": 1927
       },
       {
+        "type": "Secret Keyword",
+        "filename": "openapis/swagger.yaml",
         "hashed_secret": "bb8e48bd1e73662027a0f0b876b695d4c18f5ed4",
         "is_verified": false,
-        "line_number": 1927,
-        "type": "Secret Keyword"
+        "line_number": 1927
       },
       {
+        "type": "Secret Keyword",
+        "filename": "openapis/swagger.yaml",
         "hashed_secret": "7861ab65194de92776ab9cd06d4d7e7e1ec2c36d",
         "is_verified": false,
-        "line_number": 2007,
-        "type": "Secret Keyword"
+        "line_number": 2007
       },
       {
+        "type": "JSON Web Token",
+        "filename": "openapis/swagger.yaml",
         "hashed_secret": "d6b66ddd9ea7dbe760114bfe9a97352a5e139134",
         "is_verified": false,
-        "line_number": 2029,
-        "type": "JSON Web Token"
+        "line_number": 2029
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "openapis/swagger.yaml",
         "hashed_secret": "98c144f5ecbb4dbe575147a39698b6be1a5649dd",
         "is_verified": false,
-        "line_number": 2041,
-        "type": "Base64 High Entropy String"
+        "line_number": 2041
       }
     ],
     "tests/conftest.py": [
       {
+        "type": "Private Key",
+        "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1177,
-        "type": "Private Key"
+        "line_number": 1176
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1200,
-        "type": "Base64 High Entropy String"
+        "line_number": 1199
       }
     ],
     "tests/credentials/google/test_credentials.py": [
       {
+        "type": "Secret Keyword",
+        "filename": "tests/credentials/google/test_credentials.py",
         "hashed_secret": "22afbfecd4124e2eb0e2a79fafdf62b207a8f8c7",
         "is_verified": false,
-        "line_number": 579,
-        "type": "Secret Keyword"
+        "line_number": 579
       }
     ],
     "tests/keys/2018-05-01T21:29:02Z/jwt_private_key.pem": [
       {
+        "type": "Private Key",
+        "filename": "tests/keys/2018-05-01T21:29:02Z/jwt_private_key.pem",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1,
-        "type": "Private Key"
+        "line_number": 1
       }
     ],
     "tests/login/test_fence_login.py": [
       {
+        "type": "Secret Keyword",
+        "filename": "tests/login/test_fence_login.py",
         "hashed_secret": "d300421e208bfd0d432294de15169fd9b8975def",
         "is_verified": false,
-        "line_number": 41,
-        "type": "Secret Keyword"
+        "line_number": 41
       }
     ],
     "tests/ras/test_ras.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tests/ras/test_ras.py",
         "hashed_secret": "d9db6fe5c14dc55edd34115cdf3958845ac30882",
         "is_verified": false,
-        "line_number": 327,
-        "type": "Hex High Entropy String"
+        "line_number": 327
       }
     ],
     "tests/test-fence-config.yaml": [
       {
+        "type": "Basic Auth Credentials",
+        "filename": "tests/test-fence-config.yaml",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 31,
-        "type": "Basic Auth Credentials"
+        "line_number": 31
       }
     ]
   },
-  "version": "0.13.1",
-  "word_list": {
-    "file": null,
-    "hash": null
-  }
+  "version": "1.1.0",
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.regex.should_exclude_file",
+      "pattern": [
+        "poetry.lock"
+      ]
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    }
+  ]
 }
diff --git a/TECHDEBT.md b/TECHDEBT.md
@@ -1,22 +1 @@
 #  Tech debt
-
-### Using 'aud' claim for scopes
-- Observed: July 2020
-- Impact: (If this tech debt affected your work somehow, add a +1 here with a date and note)
-  - +1 Zoe 2020 July 15 This is an example of a +1
-  - +1 Vahid Oct 2020
-
-##### Problem:
-Fence puts OAuth2 scopes into the 'aud' claim of access tokens.
-##### Why it was done this way:
-We don't know.
-##### Why this way is problematic:
-Per RFC7519 the aud claim [is not meant for scopes](https://tools.ietf.org/html/rfc7519#section-4.1.3).
-##### What the solution might be:
-GA4GH AAI [already requires](https://github.com/ga4gh/data-security/blob/master/AAI/AAIConnectProfile.md#access_token-issued-by-broker) that a 'scope' claim be included in access tokens issued by Passport Brokers. So as of July 2020 we will put scopes in the 'scope' claim. However, this is in addition to keeping them in the 'aud' claim. Ideally we would only have the scopes in the 'scope' claim.
-##### Why we aren't already doing the above:
-Fence presently guards several endpoints (e.g. /data, signed urls, SA registration) by checking the scopes in the 'aud' claim of the JWT. This code would need to be changed.
-##### Next steps:
-Address above.
-##### Other notes:
-n/a
diff --git a/fence/auth.py b/fence/auth.py
@@ -209,7 +209,10 @@ def has_oauth(scope=None):
     scope = scope or set()
     scope.update({"openid"})
     try:
-        access_token_claims = validate_jwt(aud=scope, purpose="access")
+        access_token_claims = validate_jwt(
+            scope=scope,
+            purpose="access",
+        )
     except JWTError as e:
         raise Unauthorized("failed to validate token: {}".format(e))
     user_id = access_token_claims["sub"]

diff --git a/fence/blueprints/data/blueprint.py b/fence/blueprints/data/blueprint.py
@@ -20,7 +20,7 @@
 
 
 @blueprint.route("/<path:file_id>", methods=["DELETE"])
-@require_auth_header(aud={"data"})
+@require_auth_header(scope={"data"})
 @login_required({"data"})
 def delete_data_file(file_id):
     """
@@ -107,7 +107,7 @@ def delete_data_file(file_id):
 
 
 @blueprint.route("/upload", methods=["POST"])
-@require_auth_header(aud={"data"})
+@require_auth_header(scope={"data"})
 @login_required({"data"})
 def upload_data_file():
     """
@@ -182,7 +182,7 @@ def upload_data_file():
 
 
 @blueprint.route("/multipart/init", methods=["POST"])
-@require_auth_header(aud={"data"})
+@require_auth_header(scope={"data"})
 @login_required({"data"})
 @check_arborist_auth(resource="/data_file", method="file_upload")
 def init_multipart_upload():
@@ -213,7 +213,7 @@ def init_multipart_upload():
 
 
 @blueprint.route("/multipart/upload", methods=["POST"])
-@require_auth_header(aud={"data"})
+@require_auth_header(scope={"data"})
 @login_required({"data"})
 @check_arborist_auth(resource="/data_file", method="file_upload")
 def generate_multipart_upload_presigned_url():
@@ -247,7 +247,7 @@ def generate_multipart_upload_presigned_url():
 
 
 @blueprint.route("/multipart/complete", methods=["POST"])
-@require_auth_header(aud={"data"})
+@require_auth_header(scope={"data"})
 @login_required({"data"})
 @check_arborist_auth(resource="/data_file", method="file_upload")
 def complete_multipart_upload():