Merge pull request #744 from uc-cdis/feat/cleversafe-url

Feat/cleversafe url
uc-cdis · Mar 11, 2020 · 97d74f1 · 97d74f1
2 parents d3ce3bc + 7d78f17
commit 97d74f1
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Fence
+# Fence 
 
 [![Build Status](https://travis-ci.org/uc-cdis/fence.svg?branch=master)](https://travis-ci.org/uc-cdis/fence)
 

diff --git a/fence/__init__.py b/fence/__init__.py
@@ -171,8 +171,8 @@ def public_keys():
 
 def _check_s3_buckets(app):
     """
-    Function to ensure that all s3_buckets have a valid credential. 
-    Additionally, if there is no region it will produce a warning then trys to fetch and cache the region. 
+    Function to ensure that all s3_buckets have a valid credential.
+    Additionally, if there is no region it will produce a warning then trys to fetch and cache the region.
     """
     buckets = config.get("S3_BUCKETS") or {}
     aws_creds = config.get("AWS_CREDENTIALS") or {}
@@ -190,21 +190,25 @@ def _check_s3_buckets(app):
                     cred, bucket_name
                 )
             )
-        if not region:
-            logger.warning(
-                "WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
-                " response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
-                    bucket_name
+
+        # only require region when we're not specifying an
+        # s3-compatible endpoint URL (ex: no need for region when using cleversafe)
+        if not bucket_details.get("endpoint_url"):
+            if not region:
+                logger.warning(
+                    "WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
+                    " response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
+                        bucket_name
+                    )
                 )
-            )
-            credential = S3IndexedFileLocation.get_credential_to_access_bucket(
-                bucket_name,
-                aws_creds,
-                config.get("MAX_PRESIGNED_URL_TTL", 3600),
-                app.boto,
-            )
-            region = app.boto.get_bucket_region(bucket_name, credential)
-            config["S3_BUCKETS"][bucket_name]["region"] = region
+                credential = S3IndexedFileLocation.get_credential_to_access_bucket(
+                    bucket_name,
+                    aws_creds,
+                    config.get("MAX_PRESIGNED_URL_TTL", 3600),
+                    app.boto,
+                )
+                region = app.boto.get_bucket_region(bucket_name, credential)
+                config["S3_BUCKETS"][bucket_name]["region"] = region
 
 
 def app_config(

diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py
@@ -605,13 +605,24 @@ def get_signed_url(
         aws_creds = get_value(
             config, "AWS_CREDENTIALS", InternalError("credentials not configured")
         )
-
-        http_url = "https://{}.s3.amazonaws.com/{}".format(
-            self.parsed_url.netloc, self.parsed_url.path.strip("/")
+        s3_buckets = get_value(
+            config, "S3_BUCKETS", InternalError("buckets not configured")
         )
 
+        bucket_name = self.bucket_name()
+        bucket = s3_buckets.get(bucket_name)
+
+        if bucket and bucket.get("endpoint_url"):
+            http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
+                self.parsed_url.netloc, self.parsed_url.path.strip("/")
+            )
+        else:
+            http_url = "https://{}.s3.amazonaws.com/{}".format(
+                self.parsed_url.netloc, self.parsed_url.path.strip("/")
+            )
+
         credential = S3IndexedFileLocation.get_credential_to_access_bucket(
-            self.bucket_name(), aws_creds, expires_in
+            bucket_name, aws_creds, expires_in
         )
 
         # if it's public and we don't need to force the signed url, just return the raw
@@ -627,11 +638,15 @@ def get_signed_url(
         if aws_access_key_id == "*" or (public_data and not force_signed_url):
             return http_url
 
-        region = self.get_bucket_region()
-        if not region:
-            region = flask.current_app.boto.get_bucket_region(
-                self.parsed_url.netloc, credential
-            )
+        # only attempt to get the region when we're not specifying an
+        # s3-compatible endpoint URL (ex: no need for region when using cleversafe)
+        region = None
+        if not bucket.get("endpoint_url"):
+            region = self.get_bucket_region()
+            if not region:
+                region = flask.current_app.boto.get_bucket_region(
+                    self.parsed_url.netloc, credential
+                )
 
         user_info = _get_user_info()
 

diff --git a/fence/config-default.yaml b/fence/config-default.yaml
@@ -492,6 +492,8 @@ S3_BUCKETS: {}
 #   bucket1:
 #     cred: 'CRED1'
 #     region: 'us-east-1'
+#     # optionally you can manually specify an s3-compliant endpoint for this bucket
+#     endpoint_url: 'https://cleversafe.example.com/'
 #   bucket2:
 #     cred: 'CRED2'
 #     region: 'us-east-1'

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -461,8 +461,8 @@ def indexd_client(app, request):
             "baseid": "",
             "rev": "",
             "size": 10,
-            "file_name": "file1",
-            "urls": ["s3://bucket1/key"],
+            "file_name": "file2",
+            "urls": ["s3://bucket2/key"],
             "hashes": {},
             "acl": ["phs000178", "phs000218"],
             "form": "",

diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml
@@ -397,6 +397,7 @@ S3_BUCKETS:
     cred: 'CRED1'
   bucket2:
     cred: 'CRED2'
+    endpoint_url: 'https://cleversafe.example.com/'
     region: 'us-east-1' 
   bucket3:
     cred: 'CRED1'