Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix reading dtype from zarr dataset #72

Merged
merged 6 commits into from
May 9, 2023
Merged

Fix reading dtype from zarr dataset #72

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
78cae9e
Fix reading dtype from zarr dataset
alejoe91 May 8, 2023
b3696b9
Implement suggested changes
alejoe91 May 8, 2023
fa030ca
fix flake8
alejoe91 May 9, 2023
1ae946b
Force zarr_dtype to be str (and JSON serializable)
alejoe91 May 9, 2023
437b76f
fix logic
alejoe91 May 9, 2023
35202aa
Merge branch 'dev' into fix-dataiterator-reading
oruebel May 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/hdmf_zarr/backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,7 @@ def __setup_chunked_dataset__(cls, parent, name, data, options=None):
io_settings['dtype'] = cls.__dtypes.get(io_settings['dtype'])
try:
dset = parent.create_dataset(name, **io_settings)
dset.attrs['zarr_dtype'] = np.dtype(io_settings['dtype']).str
except Exception as exc:
raise Exception("Could not create dataset %s in %s" % (name, parent.name)) from exc
return dset
Expand Down Expand Up @@ -1099,11 +1100,18 @@ def __read_dataset(self, zarr_obj, name):
if ret is not None:
return ret

if 'zarr_dtype' not in zarr_obj.attrs:
if 'zarr_dtype' in zarr_obj.attrs:
zarr_dtype = zarr_obj.attrs['zarr_dtype']
elif hasattr(zarr_obj, 'dtype'): # Fallback for invalid files that are mssing zarr_type
zarr_dtype = zarr_obj.dtype
warnings.warn(
"Inferred dtype from zarr type. Dataset missing zarr_dtype: " + str(name) + " " + str(zarr_obj)
)
else:
raise ValueError("Dataset missing zarr_dtype: " + str(name) + " " + str(zarr_obj))

kwargs = {"attributes": self.__read_attrs(zarr_obj),
"dtype": zarr_obj.attrs['zarr_dtype'],
"dtype": zarr_dtype,
"maxshape": zarr_obj.shape,
"chunks": not (zarr_obj.shape == zarr_obj.chunks),
"source": self.source}
Expand Down