Skip to content
Permalink
Browse files
fix: populate etag / generation / metageneration properties during do…
…wnload (#488)

Populate properties during download functions via Blob._extract_headers_from_download.

Additionally, update documentation for running system tests to use Python 3.8 instead of Python 3.7 to align with nox.

Closes #490.
  • Loading branch information
daniellehanks committed Jul 6, 2021
1 parent 1ae766e commit 49ba14c9c47dbe6bc2bb45d53bbe5621c131fbcb
Showing with 37 additions and 4 deletions.
  1. +2 −2 CONTRIBUTING.rst
  2. +5 −0 google/cloud/storage/blob.py
  3. +18 −2 tests/system/test_blob.py
  4. +12 −0 tests/unit/test_blob.py
@@ -133,13 +133,13 @@ Running System Tests

- To run system tests, you can execute::

$ nox -s system-3.7
$ nox -s system-3.8
$ nox -s system-2.7

.. note::

System tests are only configured to run under Python 2.7 and
Python 3.7. For expediency, we do not run them in older versions
Python 3.8. For expediency, we do not run them in older versions
of Python 3.

This alone will not run the tests. You'll need to change some local
@@ -858,6 +858,11 @@ def _extract_headers_from_download(self, response):
self.cache_control = response.headers.get("Cache-Control", None)
self.storage_class = response.headers.get("X-Goog-Storage-Class", None)
self.content_language = response.headers.get("Content-Language", None)
self._properties["etag"] = response.headers.get("ETag", None)
self._properties["generation"] = response.headers.get("X-goog-generation", None)
self._properties["metageneration"] = response.headers.get(
"X-goog-metageneration", None
)
# 'X-Goog-Hash': 'crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==',
x_goog_hash = response.headers.get("X-Goog-Hash", "")

@@ -173,26 +173,38 @@ def test_blob_crud_w_user_project(
# Exercise 'objects.insert' w/ userProject.
blob.upload_from_filename(info["path"])
gen0 = blob.generation
etag0 = blob.etag

# Upload a second generation of the blob
blob.upload_from_string(gen1_payload)
gen1 = blob.generation
etag1 = blob.etag

blob0 = with_user_project.blob("SmallFile", generation=gen0)
blob1 = with_user_project.blob("SmallFile", generation=gen1)

# Exercise 'objects.get' w/ generation
assert with_user_project.get_blob(blob.name).generation == gen1
assert with_user_project.get_blob(blob.name, generation=gen0).generation == gen0
blob1 = with_user_project.get_blob(blob.name)
assert blob1.generation == gen1
assert blob1.etag == etag1
blob0 = with_user_project.get_blob(blob.name, generation=gen0)
assert blob0.generation == gen0
assert blob0.etag == etag0

try:
# Exercise 'objects.get' (metadata) w/ userProject.
assert blob.exists()
blob.reload()

# Exercise 'objects.get' (media) w/ userProject.
blob0 = with_user_project.blob("SmallFile", generation=gen0)
blob1 = with_user_project.blob("SmallFile", generation=gen1)
assert blob0.etag is None
assert blob1.etag is None
assert blob0.download_as_bytes() == gen0_payload
assert blob1.download_as_bytes() == gen1_payload
assert blob0.etag == etag0
assert blob1.etag == etag1

# Exercise 'objects.patch' w/ userProject.
blob0.content_language = "en"
@@ -468,10 +480,14 @@ def test_blob_download_as_text(
blob = shared_bucket.blob("MyBuffer")
payload = "Hello World"
blob.upload_from_string(payload)
etag = blob.etag
blobs_to_delete.append(blob)

blob = shared_bucket.blob("MyBuffer")
assert blob.etag is None
stored_contents = blob.download_as_text()
assert stored_contents == payload
assert blob.etag == etag


def test_blob_upload_w_gzip_encoded_download_raw(
@@ -1035,8 +1035,11 @@ def test__extract_headers_from_download_gzipped(self):
"Content-Language": "ko-kr",
"Cache-Control": "max-age=1337;public",
"Content-Encoding": "gzip",
"Etag": "kittens",
"X-Goog-Storage-Class": "STANDARD",
"X-Goog-Hash": "crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==",
"X-goog-generation": 42,
"X-goog-metageneration": 4,
},
# { "x": 5 } gzipped
content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00",
@@ -1050,6 +1053,9 @@ def test__extract_headers_from_download_gzipped(self):
self.assertEqual(blob.storage_class, "STANDARD")
self.assertEqual(blob.md5_hash, "CS9tHYTtyFntzj7B9nkkJQ==")
self.assertEqual(blob.crc32c, "4gcgLQ==")
self.assertEqual(blob.etag, "kittens")
self.assertEqual(blob.generation, 42)
self.assertEqual(blob.metageneration, 4)

def test__extract_headers_from_download_empty(self):
blob_name = "blob-name"
@@ -1064,8 +1070,11 @@ def test__extract_headers_from_download_empty(self):
"Content-Language": "en-US",
"Cache-Control": "max-age=1337;public",
"Content-Encoding": "gzip",
"Etag": "kittens",
"X-Goog-Storage-Class": "STANDARD",
"X-Goog-Hash": "crc32c=4/c+LQ==,md5=CS9tHYTt/+ntzj7B9nkkJQ==",
"X-goog-generation": 42,
"X-goog-metageneration": 4,
},
content=b"",
)
@@ -1074,6 +1083,9 @@ def test__extract_headers_from_download_empty(self):
self.assertEqual(blob.content_language, "en-US")
self.assertEqual(blob.md5_hash, "CS9tHYTt/+ntzj7B9nkkJQ==")
self.assertEqual(blob.crc32c, "4/c+LQ==")
self.assertEqual(blob.etag, "kittens")
self.assertEqual(blob.generation, 42)
self.assertEqual(blob.metageneration, 4)

def test__extract_headers_from_download_w_hash_response_header_none(self):
blob_name = "blob-name"

0 comments on commit 49ba14c

Please sign in to comment.