From 522ff0bdc820c3c973e5e832800db25e046df874 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Mon, 1 Jul 2019 14:48:33 +0300 Subject: [PATCH 1/3] Storage: Blob.exists() does not work within Batch context --- storage/google/cloud/storage/blob.py | 41 ++++++++++++++++++--- storage/tests/unit/test_blob.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index 9c89c52b9e24..9123aedd7397 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -56,6 +56,7 @@ from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._signing import generate_signed_url_v2 from google.cloud.storage._signing import generate_signed_url_v4 +from google.cloud.storage.batch import _FutureDict from google.cloud.storage.acl import ACL from google.cloud.storage.acl import ObjectACL @@ -466,18 +467,24 @@ def exists(self, client=None): query_params["fields"] = "name" try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. + if client.current_batch is not None: + target = FutureBoolResult() + result = target + else: + # We intentionally pass `_target_object=None` since fields=name + # would limit the local properties. + target, result = None, True + client._connection.api_request( method="GET", path=self.path, query_params=query_params, - _target_object=None, + _target_object=target, ) # NOTE: This will not fail immediately in a batch. However, when # Batch.finish() is called, the resulting `NotFound` will be # raised. - return True + return result except NotFound: return False @@ -2057,3 +2064,29 @@ def _add_query_parameters(base_url, name_value_pairs): query = parse_qsl(query) query.extend(name_value_pairs) return urlunsplit((scheme, netloc, path, urlencode(query), frag)) + + +class FutureBoolResult(object): + """Represents the result of future api request. + + This can be used as a target object for the future request + (which is part of batch mechanism) to get result of this request. + """ + _properties = {} + + @property + def _is_successfull(self): + """Checks if binded future request have been completed without any exception.""" + return not isinstance(self._properties, _FutureDict) + + def __bool__(self): + return self._is_successfull + + def __repr__(self): + return str(self._is_successfull) + + def __eq__(self, other): + return self._is_successfull == other + + def __ne__(self, other): + return self._is_successfull != other diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index b264ddcb8acf..4d77ae6bf0d8 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -20,6 +20,7 @@ import os import tempfile import unittest +import requests import google.cloud.storage.blob import mock @@ -27,6 +28,37 @@ from six.moves import http_client +def _make_response(status_code=http_client.OK, content=b"", headers={}): + response = requests.Response() + response.status_code = status_code + response._content = content + response.headers = headers + response.request = requests.Request() + return response + + +def _make_connection(): + from google.cloud.storage._http import Connection + + mock_conn = mock.create_autospec(Connection) + mock_conn.user_agent = "testing 1.2.3" + mock_conn._make_request.return_value = _make_response( + status_code=200, + headers={ + "content-type": "multipart/mixed; boundary=batch_pK7JBAk73-E=_AA5eFwv4m2Q=", + }, + content=b""" +--batch_pK7JBAk73-E=_AA5eFwv4m2Q= +HTTP/1.1 404 NotFound + +--batch_pK7JBAk73-E=_AA5eFwv4m2Q= +HTTP/1.1 200 OK + +--batch_pK7JBAk73-E=_AA5eFwv4m2Q=--""" + ) + return mock_conn + + def _make_credentials(): import google.auth.credentials @@ -582,6 +614,27 @@ def test_exists_miss(self): }, ) + def test_exists_miss_within_batch(self): + from google.cloud.storage.client import Client + from google.cloud.exceptions import NotFound + + client = Client(credentials=_make_credentials()) + client._base_connection = _make_connection() + + bucket = _Bucket(client) + blob1 = self._make_one("nonesuch1", bucket=bucket) + blob2 = self._make_one("nonesuch2", bucket=bucket) + + try: + with client.batch(): + bool1 = blob1.exists() + bool2 = blob2.exists() + except NotFound: + pass + + self.assertFalse(bool1) + self.assertTrue(bool2) + def test_exists_hit_w_user_project(self): BLOB_NAME = "blob-name" USER_PROJECT = "user-project-123" @@ -3311,6 +3364,7 @@ def delete_blob(self, blob_name, client=None, generation=None): class _Client(object): def __init__(self, connection): self._base_connection = connection + self.current_batch = None @property def _connection(self): From 42ca9447d99d15243800e2e00d82c925839c6280 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 10 Jul 2019 11:13:22 +0300 Subject: [PATCH 2/3] Fix assert for python-2.7 and add some tests --- storage/google/cloud/storage/blob.py | 3 +++ storage/noxfile.py | 4 ++-- storage/tests/unit/test_blob.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index 9123aedd7397..c7e6c4d50e9e 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -2082,6 +2082,9 @@ def _is_successfull(self): def __bool__(self): return self._is_successfull + def __nonzero__(self): + return self._is_successfull + def __repr__(self): return str(self._is_successfull) diff --git a/storage/noxfile.py b/storage/noxfile.py index e66f04dac0ec..5db5eff27ef1 100644 --- a/storage/noxfile.py +++ b/storage/noxfile.py @@ -134,7 +134,7 @@ def blacken(session): "tests", "docs", ) - + @nox.session(python="3.7") def docs(session): """Build the docs for this library.""" @@ -154,4 +154,4 @@ def docs(session): os.path.join("docs", "_build", "doctrees", ""), os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), - ) \ No newline at end of file + ) diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index 4d77ae6bf0d8..78afd775bdf8 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -634,6 +634,8 @@ def test_exists_miss_within_batch(self): self.assertFalse(bool1) self.assertTrue(bool2) + self.assertEqual(str(bool1), 'False') + self.assertTrue(bool1 != True) def test_exists_hit_w_user_project(self): BLOB_NAME = "blob-name" From 8d43ebc94d2f13d3e31b28a8eea2f7cb89899a7e Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 11 Jul 2019 13:06:25 +0300 Subject: [PATCH 3/3] Add one more assert and ignoring comments --- storage/tests/unit/test_blob.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index 78afd775bdf8..24c52c5fae9a 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -635,7 +635,8 @@ def test_exists_miss_within_batch(self): self.assertFalse(bool1) self.assertTrue(bool2) self.assertEqual(str(bool1), 'False') - self.assertTrue(bool1 != True) + self.assertTrue(bool1 != True) # noqa: E712 + self.assertTrue(bool1 == False) # noqa: E712 def test_exists_hit_w_user_project(self): BLOB_NAME = "blob-name"