From 74179b29f0abc25e91a2507f7c1801da70ab354f Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Wed, 18 Dec 2019 09:29:34 -0600 Subject: [PATCH 1/9] feat(cleversafe): allow specifying s3-compliant url for a bucket in cfg for signed url creation --- fence/blueprints/data/indexd.py | 19 +++++++++++++++---- fence/config-default.yaml | 2 ++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index d7895b7754..b9c3b2973f 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -594,13 +594,24 @@ def get_signed_url( aws_creds = get_value( config, "AWS_CREDENTIALS", InternalError("credentials not configured") ) - - http_url = "https://{}.s3.amazonaws.com/{}".format( - self.parsed_url.netloc, self.parsed_url.path.strip("/") + s3_buckets = get_value( + config, "S3_BUCKETS", InternalError("buckets not configured") ) + bucket = self.bucket_name() + + url_for_s3 = config["S3_BUCKETS"].get("endpoint_url") + if url_for_s3: + http_url = url_for_s3.strip("/") + "/{}".format( + self.parsed_url.path.strip("/") + ) + else: + http_url = "https://{}.s3.amazonaws.com/{}".format( + self.parsed_url.netloc, self.parsed_url.path.strip("/") + ) + credential = S3IndexedFileLocation.get_credential_to_access_bucket( - self.bucket_name(), aws_creds, expires_in + bucket, aws_creds, expires_in ) # if it's public and we don't need to force the signed url, just return the raw diff --git a/fence/config-default.yaml b/fence/config-default.yaml index 9984a14d26..d2dc734a26 100644 --- a/fence/config-default.yaml +++ b/fence/config-default.yaml @@ -483,6 +483,8 @@ AWS_CREDENTIALS: S3_BUCKETS: bucket1: cred: 'CRED1' + # optionally you can manually specify an s3-compliant endpoint for this bucket + endpoint_url: 'https://cleversafe.example.com/' bucket2: cred: 'CRED2' region: 'us-east-1' #optional but if specified avoids a call to GetBucketLocation which you may lack the AWS ACLs for. From 40c336ab3db5fcc3665c2a3ec6d42d363b8a0675 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Wed, 18 Dec 2019 09:50:29 -0600 Subject: [PATCH 2/9] rebuild --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6cecd77e0e..8e11f2f68c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Fence +# Fence [![Build Status](https://travis-ci.org/uc-cdis/fence.svg?branch=master)](https://travis-ci.org/uc-cdis/fence) From 14acf995b0e48b56bd11dac8b186e61f0f3c7493 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Wed, 18 Dec 2019 11:47:21 -0600 Subject: [PATCH 3/9] fix(endpoint-url): fix getting the endpoint url --- fence/blueprints/data/indexd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index b9c3b2973f..bcd5e389db 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -599,8 +599,9 @@ def get_signed_url( ) bucket = self.bucket_name() + current_bucket = s3_buckets.get(self.bucket_name()) - url_for_s3 = config["S3_BUCKETS"].get("endpoint_url") + url_for_s3 = current_bucket["endpoint_url"] if url_for_s3: http_url = url_for_s3.strip("/") + "/{}".format( self.parsed_url.path.strip("/") From a0c4a39a87f667d218bfbf43c9ac48610562dbf9 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Wed, 18 Dec 2019 13:00:12 -0600 Subject: [PATCH 4/9] fix(endpoint-url): fix getting the endpoint url --- fence/blueprints/data/indexd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index bcd5e389db..0193b21dbd 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -603,8 +603,8 @@ def get_signed_url( url_for_s3 = current_bucket["endpoint_url"] if url_for_s3: - http_url = url_for_s3.strip("/") + "/{}".format( - self.parsed_url.path.strip("/") + http_url = url_for_s3.strip("/") + "/{}/{}".format( + self.parsed_url.netloc, self.parsed_url.path.strip("/") ) else: http_url = "https://{}.s3.amazonaws.com/{}".format( From 2671e14180ca402d9cca9029dd09047cfe0b0386 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 6 Jan 2020 09:29:06 -0600 Subject: [PATCH 5/9] fix(endpoint-url): fix getting the endpoint url --- fence/blueprints/data/indexd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 0193b21dbd..da0201fd41 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -601,9 +601,8 @@ def get_signed_url( bucket = self.bucket_name() current_bucket = s3_buckets.get(self.bucket_name()) - url_for_s3 = current_bucket["endpoint_url"] - if url_for_s3: - http_url = url_for_s3.strip("/") + "/{}/{}".format( + if current_bucket["endpoint_url"]: + http_url = current_bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") ) else: From 5b8f159ebff82ad2efa461d11442af6a72f45468 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 6 Jan 2020 15:12:48 -0600 Subject: [PATCH 6/9] fix(endpoint-url): fix getting the endpoint url --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index da0201fd41..fac390e5a1 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -601,7 +601,7 @@ def get_signed_url( bucket = self.bucket_name() current_bucket = s3_buckets.get(self.bucket_name()) - if current_bucket["endpoint_url"]: + if "endpoint_url" in current_bucket and current_bucket["endpoint_url"]: http_url = current_bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") ) From a72bf6e1ff6c7ca0d744d15a0ad2bfae8ac83ced Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Mon, 6 Jan 2020 15:16:24 -0600 Subject: [PATCH 7/9] fix(endpoint-url): fix getting the endpoint url --- fence/blueprints/data/indexd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index fac390e5a1..1652a34c71 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -601,7 +601,7 @@ def get_signed_url( bucket = self.bucket_name() current_bucket = s3_buckets.get(self.bucket_name()) - if "endpoint_url" in current_bucket and current_bucket["endpoint_url"]: + if current_bucket and current_bucket.get("endpoint_url"): http_url = current_bucket["endpoint_url"].strip("/") + "/{}/{}".format( self.parsed_url.netloc, self.parsed_url.path.strip("/") ) From 554a2381428923e50ed33c94d123061a885bab8e Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Thu, 5 Mar 2020 13:44:31 -0600 Subject: [PATCH 8/9] chore(tests): configure tests to have a bucket with a different s3 endpoint to ensure it doesn't break existing signed url logic/unit tests --- tests/conftest.py | 4 ++-- tests/test-fence-config.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b2f3bcefd1..a02a2d6b18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -461,8 +461,8 @@ def indexd_client(app, request): "baseid": "", "rev": "", "size": 10, - "file_name": "file1", - "urls": ["s3://bucket1/key"], + "file_name": "file2", + "urls": ["s3://bucket2/key"], "hashes": {}, "acl": ["phs000178", "phs000218"], "form": "", diff --git a/tests/test-fence-config.yaml b/tests/test-fence-config.yaml index c572f39b45..8e03b97168 100644 --- a/tests/test-fence-config.yaml +++ b/tests/test-fence-config.yaml @@ -360,6 +360,7 @@ S3_BUCKETS: cred: 'CRED1' bucket2: cred: 'CRED2' + endpoint_url: 'https://cleversafe.example.com/' bucket3: cred: 'CRED1' bucket4: From 246359adb60a03321eb600b82f40bfe23c3c2edf Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Tue, 10 Mar 2020 12:31:18 -0500 Subject: [PATCH 9/9] Update fence/blueprints/data/indexd.py Co-Authored-By: Pauline Ribeyre --- fence/blueprints/data/indexd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 1652a34c71..a621215216 100644 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -598,8 +598,8 @@ def get_signed_url( config, "S3_BUCKETS", InternalError("buckets not configured") ) - bucket = self.bucket_name() - current_bucket = s3_buckets.get(self.bucket_name()) + bucket_name = self.bucket_name() + bucket = s3_buckets.get(bucket_name) if current_bucket and current_bucket.get("endpoint_url"): http_url = current_bucket["endpoint_url"].strip("/") + "/{}/{}".format(