Skip to content

Commit

Permalink
new compose surface
Browse files Browse the repository at this point in the history
  • Loading branch information
IlyaFaer committed May 14, 2020
1 parent 06b88be commit 2ddda40
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 33 deletions.
63 changes: 47 additions & 16 deletions google/cloud/storage/blob.py
Expand Up @@ -2007,12 +2007,7 @@ def make_private(self, client=None):
self.acl.save(client=client)

def compose(
self,
sources,
client=None,
timeout=_DEFAULT_TIMEOUT,
if_generation_match=None,
if_metageneration_match=None,
self, sources, client=None, timeout=_DEFAULT_TIMEOUT,
):
"""Concatenate source blobs into this one.
Expand Down Expand Up @@ -2054,18 +2049,17 @@ def compose(
query_params["userProject"] = self.user_project

source_objects = []
for index, source in enumerate(sources):
source_object = {"name": source.name}

preconditions = {}
if if_generation_match is not None:
preconditions["ifGenerationMatch"] = if_generation_match[index]
for source in sources:
if isinstance(source, dict):
source_object = {"name": source["blob"].name}

if if_metageneration_match is not None:
preconditions["ifMetagenerationMatch"] = if_metageneration_match[index]
preconditions = {}
_add_generation_match_parameters(preconditions, **source)

if preconditions:
source_object["objectPreconditions"] = preconditions
if preconditions:
source_object["objectPreconditions"] = preconditions
else:
source_object = {"name": source.name}

source_objects.append(source_object)

Expand Down Expand Up @@ -2697,3 +2691,40 @@ def _raise_for_more_than_one_none(**kwargs):
)

raise ValueError(msg)


_GENERATION_MATCH_PARAMETERS = (
("if_generation_match", "ifGenerationMatch"),
("if_generation_not_match", "ifGenerationNotMatch"),
("if_metageneration_match", "ifMetagenerationMatch"),
("if_metageneration_not_match", "ifMetagenerationNotMatch"),
("if_source_generation_match", "ifSourceGenerationMatch"),
("if_source_generation_not_match", "ifSourceGenerationNotMatch"),
("if_source_metageneration_match", "ifSourceMetagenerationMatch"),
("if_source_metageneration_not_match", "ifSourceMetagenerationNotMatch"),
)


def _add_generation_match_parameters(parameters, **match_parameters):
"""Add generation match parameters into the given parameters list.
:type parameters: list or dict
:param parameters: Parameters list or dict.
:type match_parameters: dict
:param match_parameters: if*generation*match parameters to add.
:raises: :exc:`ValueError` if ``parameters`` is not a ``list()``
or a ``dict()``.
"""
for snakecase_name, camelcase_name in _GENERATION_MATCH_PARAMETERS:
value = match_parameters.get(snakecase_name)

if value is not None:
if isinstance(parameters, list):
parameters.append((camelcase_name, value))

elif isinstance(parameters, dict):
parameters[camelcase_name] = value

else:
raise ValueError(
"`parameters` argument should be a dict() or a list()."
)
20 changes: 12 additions & 8 deletions tests/system/test_system.py
Expand Up @@ -1292,17 +1292,21 @@ def test_compose_with_generation_match(self):

with self.assertRaises(google.api_core.exceptions.PreconditionFailed):
original.compose(
[original, to_append],
if_generation_match=[6, 7],
if_metageneration_match=[8, 9],
[
{"blob": original, "if_generation_match": 6},
{"blob": to_append, "if_metageneration_match": 9},
],
)

original.compose(
[original, to_append],
if_generation_match=[original.generation, to_append.generation],
if_metageneration_match=[original.metageneration, to_append.metageneration],
[
{
"blob": original,
"if_generation_match": original.generation,
"if_metageneration_match": original.metageneration,
},
to_append,
]
)

composed = original.download_as_string()
self.assertEqual(composed, BEFORE + TO_APPEND)

Expand Down
21 changes: 12 additions & 9 deletions tests/unit/test_blob.py
Expand Up @@ -2619,23 +2619,27 @@ def test_compose_w_additional_property_changes(self):
def test_compose_w_generation_match(self):
SOURCE_1 = "source-1"
SOURCE_2 = "source-2"
SOURCE_3 = "source-3"
DESTINATION = "destination"
RESOURCE = {}
GENERATION_NUMBERS = [6, 9]
METAGENERATION_NUMBERS = [7, 1]
GENERATION_NUMBER = 6
METAGENERATION_NUMBER = 9

after = ({"status": http_client.OK}, RESOURCE)
connection = _Connection(after)
client = _Client(connection)
bucket = _Bucket(client=client)
source_1 = self._make_one(SOURCE_1, bucket=bucket)
source_2 = self._make_one(SOURCE_2, bucket=bucket)
source_3 = self._make_one(SOURCE_3, bucket=bucket)

destination = self._make_one(DESTINATION, bucket=bucket)
destination.compose(
sources=[source_1, source_2],
if_generation_match=GENERATION_NUMBERS,
if_metageneration_match=METAGENERATION_NUMBERS,
sources=[
{"blob": source_1, "if_generation_match": GENERATION_NUMBER},
{"blob": source_2, "if_metageneration_match": METAGENERATION_NUMBER},
source_3,
]
)

kw = connection._requested
Expand All @@ -2651,17 +2655,16 @@ def test_compose_w_generation_match(self):
{
"name": source_1.name,
"objectPreconditions": {
"ifGenerationMatch": GENERATION_NUMBERS[0],
"ifMetagenerationMatch": METAGENERATION_NUMBERS[0],
"ifGenerationMatch": GENERATION_NUMBER,
},
},
{
"name": source_2.name,
"objectPreconditions": {
"ifGenerationMatch": GENERATION_NUMBERS[1],
"ifMetagenerationMatch": METAGENERATION_NUMBERS[1],
"ifMetagenerationMatch": METAGENERATION_NUMBER,
},
},
{"name": source_3.name},
],
"destination": {},
},
Expand Down

0 comments on commit 2ddda40

Please sign in to comment.