Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix exception in datacube dataset update for numeric keys #983

Merged
merged 2 commits into from
Jul 6, 2020
Merged

Fix exception in datacube dataset update for numeric keys #983

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0063f22
Make 'Unsafe changes' UI more consistent, reliable
jeremyh Jul 6, 2020
8691ebc
Tell users which document failed update
jeremyh Jul 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 14 additions & 10 deletions datacube/index/_datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from datacube.model import Dataset, DatasetType
from datacube.model.utils import flatten_datasets
from datacube.utils import jsonify_document, changes, cached_property
from datacube.utils import jsonify_document, _readable_offset, changes, cached_property
from datacube.utils.changes import get_doc_changes
from . import fields

Expand Down Expand Up @@ -265,7 +265,7 @@ def can_update(self, dataset, updates_allowed=None):

return not bad_changes, good_changes, bad_changes

def update(self, dataset, updates_allowed=None):
def update(self, dataset: Dataset, updates_allowed=None):
"""
Update dataset metadata and location
:param Dataset dataset: Dataset to update
Expand All @@ -280,17 +280,21 @@ def update(self, dataset, updates_allowed=None):
_LOG.info("No changes detected for dataset %s", dataset.id)
return dataset

if not can_update:
full_message = "Unsafe changes at " + ", ".join(".".join(offset) for offset, _, _ in unsafe_changes)
raise ValueError(full_message)

_LOG.info("Updating dataset %s", dataset.id)

for offset, old_val, new_val in safe_changes:
_LOG.info("Safe change from %r to %r", old_val, new_val)
_LOG.info("Safe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

for offset, old_val, new_val in unsafe_changes:
_LOG.info("Unsafe change from %r to %r", old_val, new_val)
_LOG.warning("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

if not can_update:
raise ValueError(f"Unsafe changes in {dataset.id}: " + (
", ".join(
_readable_offset(offset)
for offset, _, _ in unsafe_changes
)
))

_LOG.info("Updating dataset %s", dataset.id)

product = self.types.get_by_name(dataset.type.name)
with self._db.begin() as transaction:
Expand Down
13 changes: 8 additions & 5 deletions datacube/index/_metadata_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,11 +111,11 @@ def can_update(self, metadata_type, allow_unsafe_updates=False):
_LOG.info("Safe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

for offset, old_val, new_val in bad_changes:
_LOG.info("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)
_LOG.warning("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

return allow_unsafe_updates or not bad_changes, good_changes, bad_changes

def update(self, metadata_type, allow_unsafe_updates=False, allow_table_lock=False):
def update(self, metadata_type: MetadataType, allow_unsafe_updates=False, allow_table_lock=False):
"""
Update a metadata type from the document. Unsafe changes will throw a ValueError by default.

Expand All @@ -137,9 +137,12 @@ def update(self, metadata_type, allow_unsafe_updates=False, allow_table_lock=Fal
return self.get_by_name(metadata_type.name)

if not can_update:
full_message = "Unsafe changes at " + ", ".join(".".join(map(str, offset))
for offset, _, _ in unsafe_changes)
raise ValueError(full_message)
raise ValueError(f"Unsafe changes in {metadata_type.name}: " + (
", ".join(
_readable_offset(offset)
for offset, _, _ in unsafe_changes
)
))

_LOG.info("Updating metadata type %s", metadata_type.name)

Expand Down
9 changes: 4 additions & 5 deletions datacube/index/_products.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,11 @@ def can_update(self, product, allow_unsafe_updates=False):
_LOG.info("Safe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

for offset, old_val, new_val in bad_changes:
_LOG.info("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)
_LOG.warning("Unsafe change in %s from %r to %r", _readable_offset(offset), old_val, new_val)

return allow_unsafe_updates or not bad_changes, good_changes, bad_changes

def update(self, product, allow_unsafe_updates=False, allow_table_lock=False):
def update(self, product: DatasetType, allow_unsafe_updates=False, allow_table_lock=False):
"""
Update a product. Unsafe changes will throw a ValueError by default.

Expand All @@ -176,13 +176,12 @@ def update(self, product, allow_unsafe_updates=False, allow_table_lock=False):
return self.get_by_name(product.name)

if not can_update:
full_message = "Unsafe changes at " + (
raise ValueError(f"Unsafe changes in {product.name}: " + (
", ".join(
_readable_offset(offset)
for offset, _, _ in unsafe_changes
)
)
raise ValueError(full_message)
))

_LOG.info("Updating product %s", product.name)

Expand Down