Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/cleversafe url #744

Merged
merged 18 commits into from
Mar 11, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Fence
# Fence

[![Build Status](https://travis-ci.org/uc-cdis/fence.svg?branch=master)](https://travis-ci.org/uc-cdis/fence)

Expand Down
36 changes: 20 additions & 16 deletions fence/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,8 @@ def public_keys():

def _check_s3_buckets(app):
"""
Function to ensure that all s3_buckets have a valid credential.
Additionally, if there is no region it will produce a warning then trys to fetch and cache the region.
Function to ensure that all s3_buckets have a valid credential.
Additionally, if there is no region it will produce a warning then trys to fetch and cache the region.
"""
buckets = config.get("S3_BUCKETS") or {}
aws_creds = config.get("AWS_CREDENTIALS") or {}
Expand All @@ -190,21 +190,25 @@ def _check_s3_buckets(app):
cred, bucket_name
)
)
if not region:
logger.warning(
"WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
" response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
bucket_name

# only require region when we're not specifying an
# s3-compatible endpoint URL (ex: no need for region when using cleversafe)
if not bucket_details.get("endpoint_url"):
if not region:
logger.warning(
"WARNING: no region for S3_BUCKET: {}. Providing the region will reduce"
" response time and avoid a call to GetBucketLocation which you make lack the AWS ACLs for.".format(
bucket_name
)
)
)
credential = S3IndexedFileLocation.get_credential_to_access_bucket(
bucket_name,
aws_creds,
config.get("MAX_PRESIGNED_URL_TTL", 3600),
app.boto,
)
region = app.boto.get_bucket_region(bucket_name, credential)
config["S3_BUCKETS"][bucket_name]["region"] = region
credential = S3IndexedFileLocation.get_credential_to_access_bucket(
bucket_name,
aws_creds,
config.get("MAX_PRESIGNED_URL_TTL", 3600),
app.boto,
)
region = app.boto.get_bucket_region(bucket_name, credential)
config["S3_BUCKETS"][bucket_name]["region"] = region


def app_config(
Expand Down
33 changes: 24 additions & 9 deletions fence/blueprints/data/indexd.py
Original file line number Diff line number Diff line change
Expand Up @@ -605,13 +605,24 @@ def get_signed_url(
aws_creds = get_value(
config, "AWS_CREDENTIALS", InternalError("credentials not configured")
)

http_url = "https://{}.s3.amazonaws.com/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/")
s3_buckets = get_value(
config, "S3_BUCKETS", InternalError("buckets not configured")
)

bucket_name = self.bucket_name()
bucket = s3_buckets.get(bucket_name)

if bucket and bucket.get("endpoint_url"):
http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/")
)
else:
http_url = "https://{}.s3.amazonaws.com/{}".format(
self.parsed_url.netloc, self.parsed_url.path.strip("/")
)

credential = S3IndexedFileLocation.get_credential_to_access_bucket(
self.bucket_name(), aws_creds, expires_in
bucket_name, aws_creds, expires_in
)

# if it's public and we don't need to force the signed url, just return the raw
Expand All @@ -627,11 +638,15 @@ def get_signed_url(
if aws_access_key_id == "*" or (public_data and not force_signed_url):
return http_url

region = self.get_bucket_region()
if not region:
region = flask.current_app.boto.get_bucket_region(
self.parsed_url.netloc, credential
)
# only attempt to get the region when we're not specifying an
# s3-compatible endpoint URL (ex: no need for region when using cleversafe)
region = None
if not bucket.get("endpoint_url"):
region = self.get_bucket_region()
if not region:
region = flask.current_app.boto.get_bucket_region(
self.parsed_url.netloc, credential
)

user_info = _get_user_info()

Expand Down
2 changes: 2 additions & 0 deletions fence/config-default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,8 @@ S3_BUCKETS: {}
# bucket1:
# cred: 'CRED1'
# region: 'us-east-1'
# # optionally you can manually specify an s3-compliant endpoint for this bucket
# endpoint_url: 'https://cleversafe.example.com/'
# bucket2:
# cred: 'CRED2'
# region: 'us-east-1'
Expand Down
4 changes: 2 additions & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,8 +461,8 @@ def indexd_client(app, request):
"baseid": "",
"rev": "",
"size": 10,
"file_name": "file1",
"urls": ["s3://bucket1/key"],
"file_name": "file2",
"urls": ["s3://bucket2/key"],
"hashes": {},
"acl": ["phs000178", "phs000218"],
"form": "",
Expand Down
1 change: 1 addition & 0 deletions tests/test-fence-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,7 @@ S3_BUCKETS:
cred: 'CRED1'
bucket2:
cred: 'CRED2'
endpoint_url: 'https://cleversafe.example.com/'
region: 'us-east-1'
bucket3:
cred: 'CRED1'
Expand Down