From ef5b33ff61a482a1fb09f1a0f6c8161fa1cbeb70 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 12 Nov 2020 22:11:04 +0100 Subject: [PATCH 1/3] n5: prevent failure on missing group metadata In the N5 spec, https://github.com/saalfeldlab/n5#file-system-specification point 3, the `{"n5": "$version"}` metadata is only expected at the root level. Datasets missing the metadata on inner groups fail to load: ``` In [43]: list(zarr.hierarchy.Group(store=zarr.n5.N5Store("ex.n5")).groups()) ... ValueError: group not found at path 'setup0' ``` This changes the group parsing logic to not fail if the metadata is missing. Suggestions welcome on how to detect the top group/array in order to special case the parsing. --- zarr/n5.py | 7 ++++++- zarr/tests/test_creation.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/zarr/n5.py b/zarr/n5.py index 053cb7164..c56d65bec 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -304,13 +304,18 @@ def invert_chunk_coords(key): def group_metadata_to_n5(group_metadata): '''Convert group metadata from zarr to N5 format.''' del group_metadata['zarr_format'] + # TODO: This should only exist at the top-level group_metadata['n5'] = '2.0.0' return group_metadata def group_metadata_to_zarr(group_metadata): '''Convert group metadata from N5 to zarr format.''' - del group_metadata['n5'] + try: + del group_metadata['n5'] + except KeyError: + # This only exists at the top level + pass group_metadata['zarr_format'] = ZARR_FORMAT return group_metadata diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 44f387c2d..d92d5aa73 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -252,6 +252,24 @@ def test_open_array(): assert (10,) == z.chunks assert_array_equal(np.full(100, fill_value=42), z[:]) + store = 'data/group.n5' + z = open_group(store, mode='w') + i = z.create_group('inner') + a = i.zeros("array", shape=100, chunks=10) + a[:] = 42 + + # Edit inner/attributes.json to not include "n5" + with open('data/group.n5/inner/attributes.json', 'w') as o: + o.write("{}") + + # Re-open + a = open_group(store)["inner"]["array"] + assert isinstance(a, Array) + assert isinstance(z.store, N5Store) + assert (100,) == a.shape + assert (10,) == a.chunks + assert_array_equal(np.full(100, fill_value=42), a[:]) + def test_empty_like(): From fb71f8b398c9721f43691a78f8dd64062958bed0 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 13 Nov 2020 08:36:27 +0100 Subject: [PATCH 2/3] Use pop rather than del Co-authored-by: jakirkham --- zarr/n5.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zarr/n5.py b/zarr/n5.py index c56d65bec..4fea4e408 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -311,11 +311,7 @@ def group_metadata_to_n5(group_metadata): def group_metadata_to_zarr(group_metadata): '''Convert group metadata from N5 to zarr format.''' - try: - del group_metadata['n5'] - except KeyError: - # This only exists at the top level - pass + group_metadata.pop('n5') group_metadata['zarr_format'] = ZARR_FORMAT return group_metadata From 3272ba44f716bf468d385c6430ab2d9429a06423 Mon Sep 17 00:00:00 2001 From: jmoore Date: Fri, 13 Nov 2020 09:09:04 +0100 Subject: [PATCH 3/3] Re-add error handling --- zarr/n5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zarr/n5.py b/zarr/n5.py index 4fea4e408..8b0f06c3a 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -311,7 +311,11 @@ def group_metadata_to_n5(group_metadata): def group_metadata_to_zarr(group_metadata): '''Convert group metadata from N5 to zarr format.''' - group_metadata.pop('n5') + try: + group_metadata.pop('n5') + except KeyError: + # This only exists at the top level + pass group_metadata['zarr_format'] = ZARR_FORMAT return group_metadata