From 3204edcb82c6636f311ae7597a68fc0b4e487a95 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 15 Apr 2020 20:10:05 +0530 Subject: [PATCH 1/5] fix(storage): anonymous credentials for private bucket --- google/cloud/storage/client.py | 14 ++++++++++++++ tests/unit/test_client.py | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index d72149fc7..6005a71d8 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -40,6 +40,8 @@ from google.cloud.storage.batch import Batch from google.cloud.storage.bucket import Bucket from google.cloud.storage.blob import Blob +from google.cloud.storage import blob +from google.cloud.storage import bucket from google.cloud.storage.hmac_key import HMACKeyMetadata from google.cloud.storage.acl import BucketACL from google.cloud.storage.acl import DefaultObjectACL @@ -122,6 +124,7 @@ def __init__( if client_options.api_endpoint: api_endpoint = client_options.api_endpoint kw_args["api_endpoint"] = api_endpoint + _override_private_url_variables(api_endpoint) if no_project: self.project = None @@ -1056,3 +1059,14 @@ def _item_to_hmac_key_metadata(iterator, item): metadata = HMACKeyMetadata(iterator.client) metadata._properties = item return metadata + + +def _override_private_url_variables(api_endpoint): + blob._DOWNLOAD_URL_TEMPLATE = api_endpoint + u"/download/storage/v1{path}?alt=media" + blob._BASE_UPLOAD_TEMPLATE = ( + api_endpoint + u"/upload/storage/v1{bucket_path}/o?uploadType=" + ) + blob._MULTIPART_URL_TEMPLATE = blob._BASE_UPLOAD_TEMPLATE + u"multipart" + blob._RESUMABLE_URL_TEMPLATE = blob._BASE_UPLOAD_TEMPLATE + u"resumable" + blob._API_ACCESS_ENDPOINT = api_endpoint + bucket._API_ACCESS_ENDPOINT = api_endpoint diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 63eeb370c..c5705bf51 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -137,17 +137,27 @@ def test_ctor_w_empty_client_options(self): ) def test_ctor_w_client_options_dict(self): + from google.cloud.storage import blob PROJECT = "PROJECT" credentials = _make_credentials() - client_options = {"api_endpoint": "https://www.foo-googleapis.com"} + api_endpoint = "https://www.foo-googleapis.com" + client_options = {"api_endpoint": api_endpoint} client = self._make_one( project=PROJECT, credentials=credentials, client_options=client_options ) + self.assertEqual(client._connection.API_BASE_URL, api_endpoint) + + self.assertEqual(blob._API_ACCESS_ENDPOINT, api_endpoint) self.assertEqual( - client._connection.API_BASE_URL, "https://www.foo-googleapis.com" + blob._DOWNLOAD_URL_TEMPLATE, + api_endpoint + u"/download/storage/v1{path}?alt=media", + ) + self.assertEqual( + blob._BASE_UPLOAD_TEMPLATE, + api_endpoint + u"/upload/storage/v1{bucket_path}/o?uploadType=", ) def test_ctor_w_client_options_object(self): From 134066e356e3b2c0c6d6f0600c76bf8e0809b222 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Apr 2020 19:09:18 +0530 Subject: [PATCH 2/5] fix(storage): use API_BASE_URL for hostname --- google/cloud/storage/blob.py | 29 +++++++++++++++--------- google/cloud/storage/client.py | 14 ------------ tests/unit/test_blob.py | 40 ++++++++++++++++++++++++---------- tests/unit/test_client.py | 12 ---------- 4 files changed, 47 insertions(+), 48 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index fb329d08d..6d36b0aa8 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -54,7 +54,6 @@ from google.cloud._helpers import _rfc3339_to_datetime from google.cloud._helpers import _to_bytes from google.cloud.exceptions import NotFound -from google.cloud.storage._helpers import _get_storage_host from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._signing import generate_signed_url_v2 @@ -69,12 +68,11 @@ from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS from google.cloud.storage.constants import STANDARD_STORAGE_CLASS -_STORAGE_HOST = _get_storage_host() _API_ACCESS_ENDPOINT = "https://storage.googleapis.com" _DEFAULT_CONTENT_TYPE = u"application/octet-stream" -_DOWNLOAD_URL_TEMPLATE = _STORAGE_HOST + u"/download/storage/v1{path}?alt=media" -_BASE_UPLOAD_TEMPLATE = _STORAGE_HOST + u"/upload/storage/v1{bucket_path}/o?uploadType=" +_DOWNLOAD_URL_TEMPLATE = u"{hostname}/download/storage/v1{path}?alt=media" +_BASE_UPLOAD_TEMPLATE = u"{hostname}/upload/storage/v1{bucket_path}/o?uploadType=" _MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"multipart" _RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"resumable" # NOTE: "acl" is also writeable but we defer ACL management to @@ -646,19 +644,24 @@ def _get_transport(self, client): client = self._require_client(client) return client._http - def _get_download_url(self): + def _get_download_url(self, client): """Get the download URL for the current blob. If the ``media_link`` has been loaded, it will be used, otherwise the URL will be constructed from the current blob's path (and possibly generation) to avoid a round trip. + :type client: :class:`~google.cloud.storage.client.Client` + :param client: The client to use. + :rtype: str :returns: The download URL for the current blob. """ name_value_pairs = [] if self.media_link is None: - base_url = _DOWNLOAD_URL_TEMPLATE.format(path=self.path) + base_url = _DOWNLOAD_URL_TEMPLATE.format( + hostname=client._connection.API_BASE_URL, path=self.path + ) if self.generation is not None: name_value_pairs.append(("generation", "{:d}".format(self.generation))) else: @@ -786,7 +789,8 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ - download_url = self._get_download_url() + client = self._require_client(client) + download_url = self._get_download_url(client) headers = _get_encryption_headers(self._encryption_key) headers["accept-encoding"] = "gzip" @@ -1016,8 +1020,10 @@ def _do_multipart_upload( transport = self._get_transport(client) info = self._get_upload_arguments(content_type) headers, object_metadata, content_type = info - - base_url = _MULTIPART_URL_TEMPLATE.format(bucket_path=self.bucket.path) + client = self._require_client(client) + base_url = _MULTIPART_URL_TEMPLATE.format( + hostname=client._connection.API_BASE_URL, bucket_path=self.bucket.path + ) name_value_pairs = [] if self.user_project is not None: @@ -1113,7 +1119,10 @@ def _initiate_resumable_upload( if extra_headers is not None: headers.update(extra_headers) - base_url = _RESUMABLE_URL_TEMPLATE.format(bucket_path=self.bucket.path) + client = self._require_client(client) + base_url = _RESUMABLE_URL_TEMPLATE.format( + hostname=client._connection.API_BASE_URL, bucket_path=self.bucket.path + ) name_value_pairs = [] if self.user_project is not None: diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 6005a71d8..d72149fc7 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -40,8 +40,6 @@ from google.cloud.storage.batch import Batch from google.cloud.storage.bucket import Bucket from google.cloud.storage.blob import Blob -from google.cloud.storage import blob -from google.cloud.storage import bucket from google.cloud.storage.hmac_key import HMACKeyMetadata from google.cloud.storage.acl import BucketACL from google.cloud.storage.acl import DefaultObjectACL @@ -124,7 +122,6 @@ def __init__( if client_options.api_endpoint: api_endpoint = client_options.api_endpoint kw_args["api_endpoint"] = api_endpoint - _override_private_url_variables(api_endpoint) if no_project: self.project = None @@ -1059,14 +1056,3 @@ def _item_to_hmac_key_metadata(iterator, item): metadata = HMACKeyMetadata(iterator.client) metadata._properties = item return metadata - - -def _override_private_url_variables(api_endpoint): - blob._DOWNLOAD_URL_TEMPLATE = api_endpoint + u"/download/storage/v1{path}?alt=media" - blob._BASE_UPLOAD_TEMPLATE = ( - api_endpoint + u"/upload/storage/v1{bucket_path}/o?uploadType=" - ) - blob._MULTIPART_URL_TEMPLATE = blob._BASE_UPLOAD_TEMPLATE + u"multipart" - blob._RESUMABLE_URL_TEMPLATE = blob._BASE_UPLOAD_TEMPLATE + u"resumable" - blob._API_ACCESS_ENDPOINT = api_endpoint - bucket._API_ACCESS_ENDPOINT = api_endpoint diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index f656e6441..18ccf4403 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -751,7 +751,9 @@ def test__get_download_url_with_media_link(self): # Set the media link on the blob blob._properties["mediaLink"] = media_link - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) self.assertEqual(download_url, media_link) def test__get_download_url_with_media_link_w_user_project(self): @@ -762,8 +764,9 @@ def test__get_download_url_with_media_link_w_user_project(self): media_link = "http://test.invalid" # Set the media link on the blob blob._properties["mediaLink"] = media_link - - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) self.assertEqual( download_url, "{}?userProject={}".format(media_link, user_project) ) @@ -774,7 +777,9 @@ def test__get_download_url_on_the_fly(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media" @@ -790,7 +795,9 @@ def test__get_download_url_on_the_fly_with_generation(self): blob._properties["generation"] = str(generation) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&generation=1493058489532987" @@ -804,7 +811,9 @@ def test__get_download_url_on_the_fly_with_user_project(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&userProject={}".format(user_project) @@ -823,7 +832,9 @@ def test__get_download_url_on_the_fly_with_kms_key_name(self): blob = self._make_one(blob_name, bucket=bucket, kms_key_name=kms_resource) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media" @@ -1003,7 +1014,8 @@ def test_download_to_file_with_failure(self): def test_download_to_file_wo_media_link(self): blob_name = "blob-name" - client = mock.Mock(spec=[u"_http"]) + client = mock.Mock(_connection=_Connection, spec=[u"_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" bucket = _Bucket(client) blob = self._make_one(blob_name, bucket=bucket) blob._do_download = mock.Mock() @@ -1312,7 +1324,8 @@ def _do_multipart_success( transport = self._mock_transport(http_client.OK, {}) # Create some mock arguments. - client = mock.Mock(_http=transport, spec=["_http"]) + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" data = b"data here hear hier" stream = io.BytesIO(data) content_type = u"application/xml" @@ -1439,7 +1452,8 @@ def _initiate_resumable_helper( transport = self._mock_transport(http_client.OK, response_headers) # Create some mock arguments and call the method under test. - client = mock.Mock(_http=transport, spec=[u"_http"]) + client = mock.Mock(_http=transport, _connection=_Connection, spec=[u"_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" data = b"hello hallo halo hi-low" stream = io.BytesIO(data) content_type = u"text/plain" @@ -1666,7 +1680,8 @@ def _do_resumable_helper( ) # Create some mock arguments and call the method under test. - client = mock.Mock(_http=transport, spec=["_http"]) + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" stream = io.BytesIO(data) content_type = u"text/html" response = blob._do_resumable_upload( @@ -1932,7 +1947,8 @@ def _create_resumable_upload_session_helper(self, origin=None, side_effect=None) # Create some mock arguments and call the method under test. content_type = u"text/plain" size = 10000 - client = mock.Mock(_http=transport, spec=[u"_http"]) + client = mock.Mock(_http=transport, _connection=_Connection, spec=[u"_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" new_url = blob.create_resumable_upload_session( content_type=content_type, size=size, origin=origin, client=client ) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index c5705bf51..c3a39351b 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -137,8 +137,6 @@ def test_ctor_w_empty_client_options(self): ) def test_ctor_w_client_options_dict(self): - from google.cloud.storage import blob - PROJECT = "PROJECT" credentials = _make_credentials() api_endpoint = "https://www.foo-googleapis.com" @@ -150,16 +148,6 @@ def test_ctor_w_client_options_dict(self): self.assertEqual(client._connection.API_BASE_URL, api_endpoint) - self.assertEqual(blob._API_ACCESS_ENDPOINT, api_endpoint) - self.assertEqual( - blob._DOWNLOAD_URL_TEMPLATE, - api_endpoint + u"/download/storage/v1{path}?alt=media", - ) - self.assertEqual( - blob._BASE_UPLOAD_TEMPLATE, - api_endpoint + u"/upload/storage/v1{bucket_path}/o?uploadType=", - ) - def test_ctor_w_client_options_object(self): from google.api_core.client_options import ClientOptions From dbd74faf364ace3fed11ff0b29b7e4b139c25353 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 17 Apr 2020 17:22:41 +0530 Subject: [PATCH 3/5] fix(storage): nit --- google/cloud/storage/blob.py | 17 ++++++----------- tests/unit/test_blob.py | 25 +++++++------------------ 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index ae2392260..6586a9064 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -648,23 +648,20 @@ def _get_transport(self, client): client = self._require_client(client) return client._http - def _get_download_url(self, client): + def _get_download_url(self): """Get the download URL for the current blob. If the ``media_link`` has been loaded, it will be used, otherwise the URL will be constructed from the current blob's path (and possibly generation) to avoid a round trip. - :type client: :class:`~google.cloud.storage.client.Client` - :param client: The client to use. - :rtype: str :returns: The download URL for the current blob. """ name_value_pairs = [] if self.media_link is None: base_url = _DOWNLOAD_URL_TEMPLATE.format( - hostname=client._connection.API_BASE_URL, path=self.path + hostname=self.client._connection.API_BASE_URL, path=self.path ) if self.generation is not None: name_value_pairs.append(("generation", "{:d}".format(self.generation))) @@ -793,8 +790,7 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ - client = self._require_client(client) - download_url = self._get_download_url(client) + download_url = self._get_download_url() headers = _get_encryption_headers(self._encryption_key) headers["accept-encoding"] = "gzip" @@ -1024,9 +1020,9 @@ def _do_multipart_upload( transport = self._get_transport(client) info = self._get_upload_arguments(content_type) headers, object_metadata, content_type = info - client = self._require_client(client) + base_url = _MULTIPART_URL_TEMPLATE.format( - hostname=client._connection.API_BASE_URL, bucket_path=self.bucket.path + hostname=self.client._connection.API_BASE_URL, bucket_path=self.bucket.path ) name_value_pairs = [] @@ -1123,9 +1119,8 @@ def _initiate_resumable_upload( if extra_headers is not None: headers.update(extra_headers) - client = self._require_client(client) base_url = _RESUMABLE_URL_TEMPLATE.format( - hostname=client._connection.API_BASE_URL, bucket_path=self.bucket.path + hostname=self.client._connection.API_BASE_URL, bucket_path=self.bucket.path ) name_value_pairs = [] diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 18ccf4403..eb6fa3f73 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -751,9 +751,7 @@ def test__get_download_url_with_media_link(self): # Set the media link on the blob blob._properties["mediaLink"] = media_link - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + download_url = blob._get_download_url() self.assertEqual(download_url, media_link) def test__get_download_url_with_media_link_w_user_project(self): @@ -764,9 +762,8 @@ def test__get_download_url_with_media_link_w_user_project(self): media_link = "http://test.invalid" # Set the media link on the blob blob._properties["mediaLink"] = media_link - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + + download_url = blob._get_download_url() self.assertEqual( download_url, "{}?userProject={}".format(media_link, user_project) ) @@ -777,9 +774,7 @@ def test__get_download_url_on_the_fly(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + download_url = blob._get_download_url() expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media" @@ -795,9 +790,7 @@ def test__get_download_url_on_the_fly_with_generation(self): blob._properties["generation"] = str(generation) self.assertIsNone(blob.media_link) - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + download_url = blob._get_download_url() expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&generation=1493058489532987" @@ -811,9 +804,7 @@ def test__get_download_url_on_the_fly_with_user_project(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + download_url = blob._get_download_url() expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&userProject={}".format(user_project) @@ -832,9 +823,7 @@ def test__get_download_url_on_the_fly_with_kms_key_name(self): blob = self._make_one(blob_name, bucket=bucket, kms_key_name=kms_resource) self.assertIsNone(blob.media_link) - client = mock.Mock(_connection=_Connection) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - download_url = blob._get_download_url(client) + download_url = blob._get_download_url() expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media" From ec458097336c1fd19dc89cd0185ec661012fcb54 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 20 Apr 2020 15:43:06 +0530 Subject: [PATCH 4/5] fix(storage): fix system test --- google/cloud/storage/blob.py | 2 +- google/cloud/storage/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 5b57ce2ff..fe2cfa54b 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -791,11 +791,11 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ - download_url = self._get_download_url() headers = _get_encryption_headers(self._encryption_key) headers["accept-encoding"] = "gzip" transport = self._get_transport(client) + download_url = self._get_download_url() try: self._do_download( transport, file_obj, download_url, headers, start, end, raw_download diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index d72149fc7..5bce4b018 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -530,7 +530,7 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): try: blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) except AttributeError: - blob = Blob.from_string(blob_or_uri) + blob = Blob.from_string(blob_or_uri, client=self) blob.download_to_file(file_obj, client=self, start=start, end=end) def list_blobs( From 6e8213be6cc81fdc05844384f24fa472e08b9b24 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 24 Apr 2020 21:00:50 +0530 Subject: [PATCH 5/5] fix(storage): revert changes --- google/cloud/storage/blob.py | 10 +++++++--- google/cloud/storage/client.py | 2 +- tests/unit/test_blob.py | 26 ++++++++++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 9f377ee32..d3416616b 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -648,20 +648,23 @@ def _get_transport(self, client): client = self._require_client(client) return client._http - def _get_download_url(self): + def _get_download_url(self, client): """Get the download URL for the current blob. If the ``media_link`` has been loaded, it will be used, otherwise the URL will be constructed from the current blob's path (and possibly generation) to avoid a round trip. + :type client: :class:`~google.cloud.storage.client.Client` + :param client: The client to use. + :rtype: str :returns: The download URL for the current blob. """ name_value_pairs = [] if self.media_link is None: base_url = _DOWNLOAD_URL_TEMPLATE.format( - hostname=self.client._connection.API_BASE_URL, path=self.path + hostname=client._connection.API_BASE_URL, path=self.path ) if self.generation is not None: name_value_pairs.append(("generation", "{:d}".format(self.generation))) @@ -790,11 +793,12 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ + client = self._require_client(client) + download_url = self._get_download_url(client) headers = _get_encryption_headers(self._encryption_key) headers["accept-encoding"] = "gzip" transport = self._get_transport(client) - download_url = self._get_download_url() try: self._do_download( transport, file_obj, download_url, headers, start, end, raw_download diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 5bce4b018..d72149fc7 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -530,7 +530,7 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): try: blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) except AttributeError: - blob = Blob.from_string(blob_or_uri, client=self) + blob = Blob.from_string(blob_or_uri) blob.download_to_file(file_obj, client=self, start=start, end=end) def list_blobs( diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index e2228eba6..bb1aa11e1 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -751,7 +751,10 @@ def test__get_download_url_with_media_link(self): # Set the media link on the blob blob._properties["mediaLink"] = media_link - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) + self.assertEqual(download_url, media_link) def test__get_download_url_with_media_link_w_user_project(self): @@ -763,7 +766,10 @@ def test__get_download_url_with_media_link_w_user_project(self): # Set the media link on the blob blob._properties["mediaLink"] = media_link - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) + self.assertEqual( download_url, "{}?userProject={}".format(media_link, user_project) ) @@ -774,7 +780,9 @@ def test__get_download_url_on_the_fly(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media" @@ -790,7 +798,9 @@ def test__get_download_url_on_the_fly_with_generation(self): blob._properties["generation"] = str(generation) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&generation=1493058489532987" @@ -804,7 +814,9 @@ def test__get_download_url_on_the_fly_with_user_project(self): blob = self._make_one(blob_name, bucket=bucket) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "fictional/o/pretend.txt?alt=media&userProject={}".format(user_project) @@ -823,7 +835,9 @@ def test__get_download_url_on_the_fly_with_kms_key_name(self): blob = self._make_one(blob_name, bucket=bucket, kms_key_name=kms_resource) self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url(client) expected_url = ( "https://storage.googleapis.com/download/storage/v1/b/" "buhkit/o/bzzz-fly.txt?alt=media"