Merge 8678c6b into 3364b22

fence/__init__.py

@@ -8,6 +8,7 @@
 from userdatamodel.driver import SQLAlchemyDriver
 
 from fence.auth import logout, build_redirect_url
+from fence.blueprints.data.indexd import S3IndexedFileLocation
 from fence.blueprints.login.utils import allowed_login_redirects, domain
 from fence.errors import UserError
 from fence.jwt import keys
@@ -41,6 +42,8 @@
 
 from cdislogging import get_logger
 
+from cdispyutils.config import get_value
+
 from gen3authz.client.arborist.client import ArboristClient
 
 # Can't read config yet. Just set to debug for now, else no handlers.
@@ -166,6 +169,44 @@ def public_keys():
         )
 
 
+def _check_s3_buckets(app):
+    """
+    Function to ensure that all s3_buckets have a valid credential. 
+    Additionally, if there is no region it will produce a warning then trys to fetch and cache the region. 
+    """
+    buckets = config.get("S3_BUCKETS", {})
+    aws_creds = config.get("AWS_CREDENTIALS", {})
+
+    for bucket_name, bucket_details in buckets.items():
+        cred = bucket_details.get("cred")
+        region = bucket_details.get("region")
+        if not cred:
+            raise ValueError(
+                "No cred for S3_BUCKET: {}. cred is required.".format(bucket_name)
+            )
+        if cred not in aws_creds and cred != "*":
+            raise ValueError(
+                "Credential {} for S3_BUCKET {} is not defined in AWS_CREDENTIALS".format(
+                    cred, bucket_name
+                )
+            )
+        if not region:
+            logger.warning(
+                "WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
+                " response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
+                    bucket_name
+                )
+            )
+            credential = S3IndexedFileLocation.get_credential_to_access_bucket(
+                bucket_name,
+                aws_creds,
+                config.get("MAX_PRESIGNED_URL_TTL", 3600),
+                app.boto,
+            )
+            region = app.boto.get_bucket_region(bucket_name, credential)
+            config["S3_BUCKETS"][bucket_name]["region"] = region
+
+
 def app_config(
     app, settings="fence.settings", root_dir=None, config_path=None, file_name=None
 ):
@@ -208,6 +249,8 @@ def app_config(
 
     _setup_oidc_clients(app)
 
+    _check_s3_buckets(app)
+
 
 def _setup_data_endpoint_and_boto(app):
     if "AWS_CREDENTIALS" in config and len(config["AWS_CREDENTIALS"]) > 0:

fence/blueprints/data/indexd.py

@@ -490,13 +490,22 @@ class S3IndexedFileLocation(IndexedFileLocation):
     """
 
     @classmethod
-    def assume_role(cls, bucket_cred, expires_in, aws_creds_config):
+    def assume_role(cls, bucket_cred, expires_in, aws_creds_config, boto=None):
+        """
+        Args:
+            bucket_cred
+            expires_in
+            aws_creds_config
+            boto (optional): provide `boto` when calling this function
+                outside of application context, to avoid errors when
+                using `flask.current_app`.
+        """
+        boto = boto or flask.current_app.boto
+
         role_arn = get_value(
             bucket_cred, "role-arn", InternalError("role-arn of that bucket is missing")
         )
-        assumed_role = flask.current_app.boto.assume_role(
-            role_arn, expires_in, aws_creds_config
-        )
+        assumed_role = boto.assume_role(role_arn, expires_in, aws_creds_config)
         cred = get_value(
             assumed_role, "Credentials", InternalError("fail to assume role")
         )
@@ -534,7 +543,9 @@ def bucket_name(self):
         return None
 
     @classmethod
-    def get_credential_to_access_bucket(cls, bucket_name, aws_creds, expires_in):
+    def get_credential_to_access_bucket(
+        cls, bucket_name, aws_creds, expires_in, boto=None
+    ):
         s3_buckets = get_value(
             config, "S3_BUCKETS", InternalError("buckets not configured")
         )
@@ -569,7 +580,7 @@ def get_credential_to_access_bucket(cls, bucket_name, aws_creds, expires_in):
                 InternalError("aws credential of that bucket is not found"),
             )
             return S3IndexedFileLocation.assume_role(
-                bucket_cred, expires_in, aws_creds_config
+                bucket_cred, expires_in, aws_creds_config, boto
             )
 
     def get_bucket_region(self):

fence/config-default.yaml

@@ -484,16 +484,21 @@ AWS_CREDENTIALS:
     aws_access_key_id: ''
     aws_secret_access_key: ''
 
+# NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a 
+# call to GetBucketLocation which you make lack the AWS ACLs for. 
 S3_BUCKETS:
   bucket1:
     cred: 'CRED1'
+    region: 'us-east-1' 
   bucket2:
     cred: 'CRED2'
-    region: 'us-east-1' #optional but if specified avoids a call to GetBucketLocation which you may lack the AWS ACLs for.
+    region: 'us-east-1' 
   bucket3:
     cred: '*'
+    region: 'us-east-1' 
   bucket4:
     cred: 'CRED1'
+    region: 'us-east-1' 
     role-arn: 'arn:aws:iam::role1'
 
 # `DATA_UPLOAD_BUCKET` specifies an S3 bucket to which data files are uploaded,

fence/resources/aws/boto_manager.py

@@ -53,6 +53,9 @@ def delete_data_file(self, bucket, guid):
             raise InternalError("Failed to delete file: {}".format(str(e)))
 
     def assume_role(self, role_arn, duration_seconds, config=None):
+        assert (
+            duration_seconds
+        ), 'assume_role() cannot be called without "duration_seconds" parameter; please check your "expires_in" parameters'
         try:
             if config and "aws_access_key_id" in config:
                 self.sts_client = client("sts", **config)

openapis/swagger.yaml

@@ -429,6 +429,8 @@ paths:
                 $ref: '#/components/schemas/SignedURL'
         '400':
           description: 'Invalid input: UUID not found or invalid location'
+        '404':
+          description: 'No location found for this file'
   '/data/upload':
     post:
       tags:

tests/test-fence-config.yaml

@@ -355,17 +355,23 @@ AWS_CREDENTIALS:
     aws_access_key_id: ''
     aws_secret_access_key: ''
 
+# NOTE: the region is optonal for s3_buckets, however it should be specified to avoid a 
+# call to GetBucketLocation which you make lack the AWS ACLs for. 
 S3_BUCKETS:
   bucket1:
     cred: 'CRED1'
   bucket2:
     cred: 'CRED2'
+    region: 'us-east-1' 
   bucket3:
     cred: 'CRED1'
+    region: 'us-east-1' 
   bucket4:
     cred: '*'
+    region: 'us-east-1' 
   bucket5:
     cred: 'CRED2'
+    region: 'us-east-1' 
     role-arn: 'arn:aws:iam::707767160287:role/bucket_reader_writer_to_cdistest-presigned-url_role'
 
 # //////////////////////////////////////////////////////////////////////////////////////