Skip to content
Permalink
Browse files
feat(storage): Add cname support for V4 signature (#72)
* feat(storage): add cname support for v4 signature

* docs(storage): comment changes

* feat(storage): address comment

* feat(storage): doc fix

* feat(storage): nit addressed

* feat(storage): add conformance tests

* feat(storage): nit

Co-authored-by: Frank Natividad <frankyn@users.noreply.github.com>
  • Loading branch information
HemangChothani and frankyn committed Mar 11, 2020
1 parent 4c1c819 commit cc853af6bf8e44e5b16e8cdfb3a275629ffb1f27
@@ -553,7 +553,7 @@ def generate_signed_url_v4(

header_names = [key.lower() for key in headers]
if "host" not in header_names:
headers["Host"] = "storage.googleapis.com"
headers["Host"] = six.moves.urllib.parse.urlparse(api_access_endpoint).netloc

if method.upper() == "RESUMABLE":
method = "POST"
@@ -362,6 +362,8 @@ def generate_signed_url(
service_account_email=None,
access_token=None,
virtual_hosted_style=False,
bucket_bound_hostname=None,
scheme="http",
):
"""Generates a signed URL for this blob.
@@ -380,6 +382,21 @@ def generate_signed_url(
amount of time, you can use this method to generate a URL that
is only valid within a certain time period.
If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,
``https`` works only if using a ``CDN``.
Example:
Generates a signed URL for this blob using bucket_bound_hostname and scheme.
>>> from google.cloud import storage
>>> client = storage.Client()
>>> bucket = client.get_bucket('my-bucket-name')
>>> blob = client.get_blob('my-blob-name')
>>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
>>> version='v4')
>>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
>>> version='v4',scheme='https') # If using ``CDN``
This is particularly useful if you don't want publicly
accessible blobs, but don't want to require users to explicitly
log in.
@@ -460,6 +477,18 @@ def generate_signed_url(
(Optional) If true, then construct the URL relative the bucket's
virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
:type bucket_bound_hostname: str
:param bucket_bound_hostname:
(Optional) If pass, then construct the URL relative to the bucket-bound hostname.
Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
See: https://cloud.google.com/storage/docs/request-endpoints#cname
:type scheme: str
:param scheme:
(Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
this value as the scheme. ``https`` will work only when using a CDN.
Defaults to ``"http"``.
:raises: :exc:`ValueError` when version is invalid.
:raises: :exc:`TypeError` when expiration is not a valid type.
:raises: :exc:`AttributeError` if credentials is not an instance
@@ -480,12 +509,21 @@ def generate_signed_url(
api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format(
bucket_name=self.bucket.name
)
resource = "/{quoted_name}".format(quoted_name=quoted_name)
elif bucket_bound_hostname:
if ":" in bucket_bound_hostname:
api_access_endpoint = bucket_bound_hostname
else:
api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
)
else:
resource = "/{bucket_name}/{quoted_name}".format(
bucket_name=self.bucket.name, quoted_name=quoted_name
)

if virtual_hosted_style or bucket_bound_hostname:
resource = "/{quoted_name}".format(quoted_name=quoted_name)

if credentials is None:
client = self._require_client(client)
credentials = client._credentials
@@ -2395,6 +2395,8 @@ def generate_signed_url(
credentials=None,
version=None,
virtual_hosted_style=False,
bucket_bound_hostname=None,
scheme="http",
):
"""Generates a signed URL for this bucket.
@@ -2413,6 +2415,20 @@ def generate_signed_url(
amount of time, you can use this method to generate a URL that
is only valid within a certain time period.
If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`,
``https`` works only if using a ``CDN``.
Example:
Generates a signed URL for this bucket using bucket_bound_hostname and scheme.
>>> from google.cloud import storage
>>> client = storage.Client()
>>> bucket = client.get_bucket('my-bucket-name')
>>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
>>> version='v4')
>>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld',
>>> version='v4',scheme='https') # If using ``CDN``
This is particularly useful if you don't want publicly
accessible buckets, but don't want to require users to explicitly
log in.
@@ -2462,6 +2478,18 @@ def generate_signed_url(
(Optional) If true, then construct the URL relative the bucket's
virtual hostname, e.g., '<bucket-name>.storage.googleapis.com'.
:type bucket_bound_hostname: str
:param bucket_bound_hostname:
(Optional) If pass, then construct the URL relative to the bucket-bound hostname.
Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'.
See: https://cloud.google.com/storage/docs/request-endpoints#cname
:type scheme: str
:param scheme:
(Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use
this value as the scheme. ``https`` will work only when using a CDN.
Defaults to ``"http"``.
:raises: :exc:`ValueError` when version is invalid.
:raises: :exc:`TypeError` when expiration is not a valid type.
:raises: :exc:`AttributeError` if credentials is not an instance
@@ -2480,10 +2508,19 @@ def generate_signed_url(
api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format(
bucket_name=self.name
)
resource = "/"
elif bucket_bound_hostname:
if ":" in bucket_bound_hostname:
api_access_endpoint = bucket_bound_hostname
else:
api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
)
else:
resource = "/{bucket_name}".format(bucket_name=self.name)

if virtual_hosted_style or bucket_bound_hostname:
resource = "/"

if credentials is None:
client = self._require_client(client)
credentials = client._credentials
@@ -778,13 +778,15 @@ def dummy_service_account():
return _DUMMY_SERVICE_ACCOUNT


def _run_conformance_test(resource, test_data):
def _run_conformance_test(
resource, test_data, api_access_endpoint="https://storage.googleapis.com"
):
credentials = dummy_service_account()

url = Test_generate_signed_url_v4._call_fut(
credentials,
resource,
expiration=test_data["expiration"],
api_access_endpoint=api_access_endpoint,
method=test_data["method"],
_request_timestamp=test_data["timestamp"],
headers=test_data.get("headers"),
@@ -802,14 +804,31 @@ def test_conformance_client(test_data):

@pytest.mark.parametrize("test_data", _BUCKET_TESTS)
def test_conformance_bucket(test_data):
resource = "/{}".format(test_data["bucket"])
_run_conformance_test(resource, test_data)
if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME":
api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=test_data["scheme"],
bucket_bound_hostname=test_data["bucketBoundHostname"],
)
resource = "/"
_run_conformance_test(resource, test_data, api_access_endpoint)
else:
resource = "/{}".format(test_data["bucket"])
_run_conformance_test(resource, test_data)


@pytest.mark.parametrize("test_data", _BLOB_TESTS)
def test_conformance_blob(test_data):
resource = "/{}/{}".format(test_data["bucket"], test_data["object"])
_run_conformance_test(resource, test_data)
if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME":
api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=test_data["scheme"],
bucket_bound_hostname=test_data["bucketBoundHostname"],
)
resource = "/{}".format(test_data["object"])
_run_conformance_test(resource, test_data, api_access_endpoint)
else:

resource = "/{}/{}".format(test_data["bucket"], test_data["object"])
_run_conformance_test(resource, test_data)


def _make_credentials(signer_email=None):
@@ -400,6 +400,8 @@ def _generate_signed_url_helper(
access_token=None,
service_account_email=None,
virtual_hosted_style=False,
bucket_bound_hostname=None,
scheme="http",
):
from six.moves.urllib import parse
from google.cloud._helpers import UTC
@@ -444,6 +446,7 @@ def _generate_signed_url_helper(
access_token=access_token,
service_account_email=service_account_email,
virtual_hosted_style=virtual_hosted_style,
bucket_bound_hostname=bucket_bound_hostname,
)

self.assertEqual(signed_uri, signer.return_value)
@@ -460,11 +463,20 @@ def _generate_signed_url_helper(
expected_api_access_endpoint = "https://{}.storage.googleapis.com".format(
bucket.name
)
expected_resource = "/{}".format(quoted_name)
elif bucket_bound_hostname:
if ":" in bucket_bound_hostname:
expected_api_access_endpoint = bucket_bound_hostname
else:
expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
)
else:
expected_api_access_endpoint = api_access_endpoint
expected_resource = "/{}/{}".format(bucket.name, quoted_name)

if virtual_hosted_style or bucket_bound_hostname:
expected_resource = "/{}".format(quoted_name)

if encryption_key is not None:
expected_headers = headers or {}
if effective_version == "v2":
@@ -619,6 +631,14 @@ def test_generate_signed_url_v4_w_csek_and_headers(self):
def test_generate_signed_url_v4_w_virtual_hostname(self):
self._generate_signed_url_v4_helper(virtual_hosted_style=True)

def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self):
self._generate_signed_url_v4_helper(
bucket_bound_hostname="http://cdn.example.com"
)

def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self):
self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com")

def test_generate_signed_url_v4_w_credentials(self):
credentials = object()
self._generate_signed_url_v4_helper(credentials=credentials)
@@ -2779,6 +2779,8 @@ def _generate_signed_url_helper(
credentials=None,
expiration=None,
virtual_hosted_style=False,
bucket_bound_hostname=None,
scheme="http",
):
from six.moves.urllib import parse
from google.cloud._helpers import UTC
@@ -2814,6 +2816,7 @@ def _generate_signed_url_helper(
query_parameters=query_parameters,
version=version,
virtual_hosted_style=virtual_hosted_style,
bucket_bound_hostname=bucket_bound_hostname,
)

self.assertEqual(signed_uri, signer.return_value)
@@ -2827,11 +2830,20 @@ def _generate_signed_url_helper(
expected_api_access_endpoint = "https://{}.storage.googleapis.com".format(
bucket_name
)
expected_resource = "/"
elif bucket_bound_hostname:
if ":" in bucket_bound_hostname:
expected_api_access_endpoint = bucket_bound_hostname
else:
expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format(
scheme=scheme, bucket_bound_hostname=bucket_bound_hostname
)
else:
expected_api_access_endpoint = api_access_endpoint
expected_resource = "/{}".format(parse.quote(bucket_name))

if virtual_hosted_style or bucket_bound_hostname:
expected_resource = "/"

expected_kwargs = {
"resource": expected_resource,
"expiration": expiration,
@@ -2967,6 +2979,14 @@ def test_generate_signed_url_v4_w_credentials(self):
def test_generate_signed_url_v4_w_virtual_hostname(self):
self._generate_signed_url_v4_helper(virtual_hosted_style=True)

def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self):
self._generate_signed_url_v4_helper(
bucket_bound_hostname="http://cdn.example.com"
)

def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self):
self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com")


class _Connection(object):
_delete_bucket = False
@@ -118,5 +118,55 @@
"expiration": 10,
"timestamp": "20190201T090000Z",
"expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b"
},

{
"description": "HTTP Bucket Bound Hostname Support",
"bucket": "test-bucket",
"object": "test-object",
"method": "GET",
"expiration": 10,
"timestamp": "20190201T090000Z",
"expectedUrl": "http://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56",
"scheme": "http",
"urlStyle": "BUCKET_BOUND_HOSTNAME",
"bucketBoundHostname": "mydomain.tld"
},

{
"description": "HTTPS Bucket Bound Hostname Support",
"bucket": "test-bucket",
"object": "test-object",
"method": "GET",
"expiration": 10,
"timestamp": "20190201T090000Z",
"expectedUrl": "https://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56",
"scheme": "https",
"urlStyle": "BUCKET_BOUND_HOSTNAME",
"bucketBoundHostname": "mydomain.tld"
},

{
"description": "HTTP Bucket Bound Hostname Support",
"bucket": "test-bucket",
"method": "GET",
"expiration": 10,
"timestamp": "20190201T090000Z",
"expectedUrl": "http://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf",
"scheme": "http",
"urlStyle": "BUCKET_BOUND_HOSTNAME",
"bucketBoundHostname": "mydomain.tld"
},

{
"description": "HTTPS Bucket Bound Hostname Support",
"bucket": "test-bucket",
"method": "GET",
"expiration": 10,
"timestamp": "20190201T090000Z",
"expectedUrl": "https://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf",
"scheme": "https",
"urlStyle": "BUCKET_BOUND_HOSTNAME",
"bucketBoundHostname": "mydomain.tld"
}
]

0 comments on commit cc853af

Please sign in to comment.