merge master

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2021-06-09T22:00:09Z",
+  "generated_at": "2021-06-10T21:56:24Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -165,14 +165,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1176
+        "line_number": 1174
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1199
+        "line_number": 1197
       }
     ],
     "tests/credentials/google/test_credentials.py": [

fence/blueprints/data/indexd.py

@@ -647,10 +647,8 @@ def assume_role(cls, bucket_cred, expires_in, aws_creds_config, boto=None):
 
         # checking fence config if aws session can be longer than one hour
         role_cache_increase = 0
-        if flask.current_app.config["MAX_ROLE_SESSION_INCREASE"]:
-            role_cache_increase = int(
-                flask.current_app.config["ASSUME_ROLE_CACHE_SECONDS"]
-            )
+        if config["MAX_ROLE_SESSION_INCREASE"]:
+            role_cache_increase = int(config["ASSUME_ROLE_CACHE_SECONDS"])
 
         assumed_role = boto.assume_role(
             role_arn,

fence/resources/aws/boto_manager.py

@@ -13,6 +13,9 @@ class BotoManager(object):
 
     URL_EXPIRATION_DEFAULT = 1800  # 30 minutes
     URL_EXPIRATION_MAX = 86400  # 1 day
+    AWS_ASSUME_ROLE_MIN_EXPIRATION = (
+        900  # minimum time for aws assume role is 900 seconds as per boto docs
+    )
 
     def __init__(self, config, logger):
         self.sts_client = client("sts", **config)
@@ -66,6 +69,9 @@ def assume_role(self, role_arn, duration_seconds, config=None):
             if config and "aws_access_key_id" in config:
                 self.sts_client = client("sts", **config)
             session_name_postfix = uuid.uuid4()
+            duration_seconds = max(
+                self.AWS_ASSUME_ROLE_MIN_EXPIRATION, duration_seconds
+            )  # Since minimum time for aws assume role is 900 seconds as per boto docs
             return self.sts_client.assume_role(
                 RoleArn=role_arn,
                 DurationSeconds=duration_seconds,

tests/conftest.py

@@ -59,17 +59,6 @@ def mock_get_bucket_location(self, bucket, config):
     return "us-east-1"
 
 
-@mock_sts
-def mock_assume_role(self, role_arn, duration_seconds, config=None):
-    sts_client = client("sts", **config)
-    session_name_postfix = uuid.uuid4()
-    return sts_client.assume_role(
-        RoleArn=role_arn,
-        DurationSeconds=duration_seconds,
-        RoleSessionName="{}-{}".format("gen3", session_name_postfix),
-    )
-
-
 @pytest.fixture(scope="session")
 def claims_refresh():
     new_claims = tests.utils.default_claims()
@@ -140,20 +129,15 @@ def mock_functions(self):
             "fence.resources.aws.boto_manager.BotoManager.get_bucket_region",
             mock_get_bucket_location,
         )
-        self.assume_role_patcher = patch(
-            "fence.resources.aws.boto_manager.BotoManager.assume_role", mock_assume_role
-        )
         self.patcher.start()
         self.auth_patcher.start()
         self.boto_patcher.start()
-        self.assume_role_patcher.start()
         self.additional_patchers = []
 
     def unmock_functions(self):
         self.patcher.stop()
         self.auth_patcher.stop()
         self.boto_patcher.stop()
-        self.assume_role_patcher.stop()
         for patcher in self.additional_patchers:
             patcher.stop()
 
@@ -475,6 +459,20 @@ def indexd_client(app, request):
             "created_date": "",
             "updated_date": "",
         }
+    elif protocol == "s3_assume_role":
+        record = {
+            "did": "",
+            "baseid": "",
+            "rev": "",
+            "size": 10,
+            "file_name": "file1",
+            "urls": ["s3://bucket5/key"],
+            "hashes": {},
+            "metadata": {"acls": "phs000178,phs000218"},
+            "form": "",
+            "created_date": "",
+            "updated_date": "",
+        }
     elif protocol == "no_urls":
         record = {
             "did": "",

tests/data/test_data.py

@@ -1074,6 +1074,87 @@ def test_indexd_download_with_uploader_authorized(
     assert response.status_code == 200
 
 
+@pytest.mark.parametrize("indexd_client", ["s3_assume_role"], indirect=True)
+@pytest.mark.parametrize("presigned_url_expires_in", [100, 1000])
+@pytest.mark.parametrize(
+    "test_max_role_session_increase, test_assume_role_cache_seconds",
+    [(True, 100), (True, 1800), (False, 0)],
+)
+def test_assume_role_time_limit(
+    client,
+    user_client,
+    kid,
+    rsa_private_key,
+    test_max_role_session_increase,
+    test_assume_role_cache_seconds,
+    presigned_url_expires_in,
+    indexd_client,
+    monkeypatch,
+):
+    """
+    Test ``GET /data/download/1`` accessing data from bucket by assuming role.
+    """
+
+    fence.S3IndexedFileLocation._assume_role_cache.clear()
+
+    monkeypatch.setitem(
+        config, "MAX_ROLE_SESSION_INCREASE", test_max_role_session_increase
+    )
+    monkeypatch.setitem(
+        config, "ASSUME_ROLE_CACHE_SECONDS", test_assume_role_cache_seconds
+    )
+    duration_in_function = 0
+
+    def mock_sts_client_assume_role(RoleArn, DurationSeconds, RoleSessionName=None):
+        nonlocal duration_in_function
+        duration_in_function = DurationSeconds
+        return {
+            "Credentials": {
+                "AccessKeyId": "",
+                "SecretAccessKey": "",
+                "SessionToken": "",
+                "Expiration": datetime.now() + timedelta(seconds=DurationSeconds),
+            },
+            "AssumedRoleUser": {"AssumedRoleId": "", "Arn": RoleArn},
+        }
+
+    with patch("fence.resources.aws.boto_manager.client") as mocked_sts_client:
+        mocked_sts_client.return_value = MagicMock(
+            assume_role=mock_sts_client_assume_role
+        )
+        indexed_file_location = indexd_client["indexed_file_location"]
+        path = "/data/download/1"
+        query_string = {
+            "protocol": indexed_file_location,
+            "expires_in": presigned_url_expires_in,
+        }
+        headers = {
+            "Authorization": "Bearer "
+            + jwt.encode(
+                utils.authorized_download_context_claims(
+                    user_client.username, user_client.user_id
+                ),
+                key=rsa_private_key,
+                headers={"kid": kid},
+                algorithm="RS256",
+            ).decode("utf-8")
+        }
+        response = client.get(path, headers=headers, query_string=query_string)
+
+    AWS_ASSUME_ROLE_MIN_EXPIRATION = 900
+
+    buffered_expires_in = presigned_url_expires_in
+    if config["MAX_ROLE_SESSION_INCREASE"]:
+        buffered_expires_in += int(config["ASSUME_ROLE_CACHE_SECONDS"])
+    buffered_expires_in = max(buffered_expires_in, AWS_ASSUME_ROLE_MIN_EXPIRATION)
+
+    assert response.status_code == 200
+    assert duration_in_function == buffered_expires_in  # assume role duration
+    assert (
+        "X-Amz-Expires=" + str(presigned_url_expires_in) + "&" in response.json["url"]
+    )  # Signed url duration
+
+
 def test_assume_role_cache(
     client,
     oauth_client,