Skip to content
Permalink
Browse files
feat: add support for Etag headers on reads (#489)
Support conditional requests based on ETag for read operations (`reload`, `exists`, `download_*`). My own testing seems to indicate that the JSON API does not support ETag If-Match/If-None-Match headers on modify requests (`patch`, `delete`, etc.), please correct me if I am mistaken.

This part two of #451. Part one in #488.
Fixes #451 🦕
  • Loading branch information
daniellehanks committed Jul 8, 2021
1 parent 49ba14c commit 741d3fda4e4280022cede29ebeb7c2ea09e73b6f
@@ -1,17 +1,31 @@
Conditional Requests Via Generation / Metageneration Preconditions
==================================================================
Conditional Requests Via ETag / Generation / Metageneration Preconditions
=========================================================================

Preconditions tell Cloud Storage to only perform a request if the
:ref:`generation <concept-generation>` or
:ref:`ETag <concept-etag>`, :ref:`generation <concept-generation>`, or
:ref:`metageneration <concept-metageneration>` number of the affected object
meets your precondition criteria. These checks of the generation and
meets your precondition criteria. These checks of the ETag, generation, and
metageneration numbers ensure that the object is in the expected state,
allowing you to perform safe read-modify-write updates and conditional
operations on objects

Concepts
--------

.. _concept-etag:

ETag
::::::::::::::

An ETag is returned as part of the response header whenever a resource is
returned, as well as included in the resource itself. Users should make no
assumptions about the value used in an ETag except that it changes whenever the
underlying data changes, per the
`specification <https://tools.ietf.org/html/rfc7232#section-2.3>`_

The ``ETag`` attribute is set by the GCS back-end, and is read-only in the
client library.

.. _concept-metageneration:

Metageneration
@@ -59,6 +73,32 @@ See also
Conditional Parameters
----------------------

.. _using-if-etag-match:

Using ``if_etag_match``
:::::::::::::::::::::::::::::

Passing the ``if_etag_match`` parameter to a method which retrieves a
blob resource (e.g.,
:meth:`Blob.reload <google.cloud.storage.blob.Blob.reload>`)
makes the operation conditional on whether the blob's current ``ETag`` matches
the given value. This parameter is not supported for modification (e.g.,
:meth:`Blob.update <google.cloud.storage.blob.Blob.update>`).


.. _using-if-etag-not-match:

Using ``if_etag_not_match``
:::::::::::::::::::::::::::::

Passing the ``if_etag_not_match`` parameter to a method which retrieves a
blob resource (e.g.,
:meth:`Blob.reload <google.cloud.storage.blob.Blob.reload>`)
makes the operation conditional on whether the blob's current ``ETag`` matches
the given value. This parameter is not supported for modification (e.g.,
:meth:`Blob.update <google.cloud.storage.blob.Blob.update>`).


.. _using-if-generation-match:

Using ``if_generation_match``
@@ -22,6 +22,7 @@
from datetime import datetime
import os

from six import string_types
from six.moves.urllib.parse import urlsplit
from google import resumable_media
from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -34,6 +35,12 @@

_DEFAULT_STORAGE_HOST = u"https://storage.googleapis.com"

# etag match parameters in snake case and equivalent header
_ETAG_MATCH_PARAMETERS = (
("if_etag_match", "If-Match"),
("if_etag_not_match", "If-None-Match"),
)

# generation match parameters in camel and snake cases
_GENERATION_MATCH_PARAMETERS = (
("if_generation_match", "ifGenerationMatch"),
@@ -147,6 +154,8 @@ def reload(
self,
client=None,
projection="noAcl",
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -168,6 +177,12 @@ def reload(
Defaults to ``'noAcl'``. Specifies the set of
properties to return.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match: (Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]])
:param if_etag_not_match: (Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -205,10 +220,14 @@ def reload(
if_metageneration_match=if_metageneration_match,
if_metageneration_not_match=if_metageneration_not_match,
)
headers = self._encryption_headers()
_add_etag_match_headers(
headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
)
api_response = client._get_resource(
self.path,
query_params=query_params,
headers=self._encryption_headers(),
headers=headers,
timeout=timeout,
retry=retry,
_target_object=self,
@@ -384,8 +403,7 @@ def update(


def _scalar_property(fieldname):
"""Create a property descriptor around the :class:`_PropertyMixin` helpers.
"""
"""Create a property descriptor around the :class:`_PropertyMixin` helpers."""

def _getter(self):
"""Scalar property getter."""
@@ -449,6 +467,24 @@ def _convert_to_timestamp(value):
return mtime


def _add_etag_match_headers(headers, **match_parameters):
"""Add generation match parameters into the given parameters list.
:type headers: dict
:param headers: Headers dict.
:type match_parameters: dict
:param match_parameters: if*etag*match parameters to add.
"""
for snakecase_name, header_name in _ETAG_MATCH_PARAMETERS:
value = match_parameters.get(snakecase_name)

if value is not None:
if isinstance(value, string_types):
value = [value]
headers[header_name] = ", ".join(value)


def _add_generation_match_parameters(parameters, **match_parameters):
"""Add generation match parameters into the given parameters list.
@@ -59,6 +59,7 @@
from google.cloud._helpers import _rfc3339_nanos_to_datetime
from google.cloud._helpers import _to_bytes
from google.cloud.exceptions import NotFound
from google.cloud.storage._helpers import _add_etag_match_headers
from google.cloud.storage._helpers import _add_generation_match_parameters
from google.cloud.storage._helpers import _PropertyMixin
from google.cloud.storage._helpers import _scalar_property
@@ -634,6 +635,8 @@ def generate_signed_url(
def exists(
self,
client=None,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -651,6 +654,14 @@ def exists(
(Optional) The client to use. If not passed, falls back to the
``client`` stored on the blob's bucket.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -692,12 +703,19 @@ def exists(
if_metageneration_match=if_metageneration_match,
if_metageneration_not_match=if_metageneration_not_match,
)

headers = {}
_add_etag_match_headers(
headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
)

try:
# We intentionally pass `_target_object=None` since fields=name
# would limit the local properties.
client._get_resource(
self.path,
query_params=query_params,
headers=headers,
timeout=timeout,
retry=retry,
_target_object=None,
@@ -1002,6 +1020,8 @@ def download_to_file(
start=None,
end=None,
raw_download=False,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -1057,6 +1077,14 @@ def download_to_file(
:param raw_download:
(Optional) If true, download the object without any expansion.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -1121,6 +1149,8 @@ def download_to_file(
start=start,
end=end,
raw_download=raw_download,
if_etag_match=if_etag_match,
if_etag_not_match=if_etag_not_match,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
@@ -1137,6 +1167,8 @@ def download_to_filename(
start=None,
end=None,
raw_download=False,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -1168,6 +1200,14 @@ def download_to_filename(
:param raw_download:
(Optional) If true, download the object without any expansion.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -1233,6 +1273,8 @@ def download_to_filename(
start=start,
end=end,
raw_download=raw_download,
if_etag_match=if_etag_match,
if_etag_not_match=if_etag_not_match,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
@@ -1260,6 +1302,8 @@ def download_as_bytes(
start=None,
end=None,
raw_download=False,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -1288,6 +1332,14 @@ def download_as_bytes(
:param raw_download:
(Optional) If true, download the object without any expansion.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -1355,6 +1407,8 @@ def download_as_bytes(
start=start,
end=end,
raw_download=raw_download,
if_etag_match=if_etag_match,
if_etag_not_match=if_etag_not_match,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
@@ -1371,6 +1425,8 @@ def download_as_string(
start=None,
end=None,
raw_download=False,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -1401,6 +1457,14 @@ def download_as_string(
:param raw_download:
(Optional) If true, download the object without any expansion.
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -1460,6 +1524,8 @@ def download_as_string(
start=start,
end=end,
raw_download=raw_download,
if_etag_match=if_etag_match,
if_etag_not_match=if_etag_not_match,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
@@ -1475,6 +1541,8 @@ def download_as_text(
end=None,
raw_download=False,
encoding=None,
if_etag_match=None,
if_etag_not_match=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
@@ -1507,6 +1575,14 @@ def download_as_text(
downloaded bytes. Defaults to the ``charset`` param of
attr:`content_type`, or else to "utf-8".
:type if_etag_match: Union[str, Set[str]]
:param if_etag_match:
(Optional) See :ref:`using-if-etag-match`
:type if_etag_not_match: Union[str, Set[str]]
:param if_etag_not_match:
(Optional) See :ref:`using-if-etag-not-match`
:type if_generation_match: long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
@@ -1558,6 +1634,8 @@ def download_as_text(
start=start,
end=end,
raw_download=raw_download,
if_etag_match=if_etag_match,
if_etag_not_match=if_etag_not_match,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,

0 comments on commit 741d3fd

Please sign in to comment.