Skip to content

Commit

Permalink
style: pre-commit fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@pre-commit-ci
pre-commit-ci[bot] committed May 27, 2024
1 parent 23b83b0 commit 9b29fbf
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 31 deletions.
92 changes: 63 additions & 29 deletions src/n5py/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
n5py: The N5 file format implemented in Python
"""


from __future__ import annotations

from ._version import version as __version__
Expand All @@ -18,16 +17,21 @@
import os
import struct
import sys
from typing import Any, Dict, Optional, cast
import warnings
from typing import Any, Dict, Optional, cast

import numpy as np
from numcodecs.abc import Codec
from numcodecs.compat import ndarray_copy
from numcodecs.registry import get_codec, register_codec

from zarr.meta import ZARR_FORMAT, json_dumps, json_loads
from zarr.storage import FSStore, NestedDirectoryStore, _prog_ckey, _prog_number, normalize_storage_path
from zarr.storage import (
FSStore,
NestedDirectoryStore,
_prog_ckey,
_prog_number,
normalize_storage_path,
)
from zarr.storage import array_meta_key as zarr_array_meta_key
from zarr.storage import attrs_key as zarr_attrs_key
from zarr.storage import group_meta_key as zarr_group_meta_key
Expand Down Expand Up @@ -94,7 +98,9 @@ def __getitem__(self, key: str) -> bytes:
elif key.endswith(zarr_array_meta_key):
key_new = key.replace(zarr_array_meta_key, n5_attrs_key)
top_level = key == zarr_array_meta_key
value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level)
value = array_metadata_to_zarr(
self._load_n5_attrs(key_new), top_level=top_level
)
return json_dumps(value)

elif key.endswith(zarr_attrs_key):
Expand Down Expand Up @@ -127,7 +133,9 @@ def __setitem__(self, key: str, value: Any):
key_new = key.replace(zarr_array_meta_key, n5_attrs_key)
top_level = key == zarr_array_meta_key
n5_attrs = self._load_n5_attrs(key_new)
n5_attrs.update(**array_metadata_to_n5(json_loads(value), top_level=top_level))
n5_attrs.update(
**array_metadata_to_n5(json_loads(value), top_level=top_level)
)
value = json_dumps(n5_attrs)

elif key.endswith(zarr_attrs_key):
Expand All @@ -138,7 +146,9 @@ def __setitem__(self, key: str, value: Any):

for k in n5_keywords:
if k in zarr_attrs:
warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning)
warnings.warn(
f"Attribute {k} is a reserved N5 keyword", UserWarning
)

# remove previous user attributes
for k in list(n5_attrs.keys()):
Expand Down Expand Up @@ -272,11 +282,10 @@ def _is_array(self, path: str):
def _contains_attrs(self, path: str):
if path is None:
attrs_key = n5_attrs_key
elif not path.endswith(n5_attrs_key):
attrs_key = os.path.join(path, n5_attrs_key)
else:
if not path.endswith(n5_attrs_key):
attrs_key = os.path.join(path, n5_attrs_key)
else:
attrs_key = path
attrs_key = path

attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key))
return len(attrs) > 0
Expand Down Expand Up @@ -379,7 +388,9 @@ def __getitem__(self, key: str) -> bytes:
elif key.endswith(zarr_array_meta_key):
key_new = key.replace(zarr_array_meta_key, self._array_meta_key)
top_level = key == zarr_array_meta_key
value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level)
value = array_metadata_to_zarr(
self._load_n5_attrs(key_new), top_level=top_level
)
return json_dumps(value)

elif key.endswith(zarr_attrs_key):
Expand Down Expand Up @@ -412,7 +423,9 @@ def __setitem__(self, key: str, value: Any):
key_new = key.replace(zarr_array_meta_key, self._array_meta_key)
top_level = key == zarr_array_meta_key
n5_attrs = self._load_n5_attrs(key_new)
n5_attrs.update(**array_metadata_to_n5(json_loads(value), top_level=top_level))
n5_attrs.update(
**array_metadata_to_n5(json_loads(value), top_level=top_level)
)

value = json_dumps(n5_attrs)

Expand All @@ -424,7 +437,9 @@ def __setitem__(self, key: str, value: Any):

for k in n5_keywords:
if k in zarr_attrs.keys():
warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning)
warnings.warn(
f"Attribute {k} is a reserved N5 keyword", UserWarning
)

# replace previous user attributes
for k in list(n5_attrs.keys()):
Expand Down Expand Up @@ -552,11 +567,10 @@ def _is_array(self, path: Optional[str]):
def _contains_attrs(self, path: Optional[str]):
if path is None:
attrs_key = self._attrs_key
elif not path.endswith(self._attrs_key):
attrs_key = os.path.join(path, self._attrs_key)
else:
if not path.endswith(self._attrs_key):
attrs_key = os.path.join(path, self._attrs_key)
else:
attrs_key = path
attrs_key = path

attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key))
return len(attrs) > 0
Expand Down Expand Up @@ -599,7 +613,9 @@ def group_metadata_to_zarr(group_metadata: Dict[str, Any]) -> Dict[str, Any]:
return group_metadata


def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dict[str, Any]:
def array_metadata_to_n5(
array_metadata: Dict[str, Any], top_level=False
) -> Dict[str, Any]:
"""Convert array metadata from zarr to N5 format. If the `top_level` keyword argument is True,
then the `N5` : N5_FORMAT key : value pair will be inserted into the metadata."""

Expand All @@ -611,14 +627,19 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic
try:
dtype = np.dtype(array_metadata["dataType"])
except TypeError:
raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5")
raise TypeError(
f"Data type {array_metadata['dataType']} is not supported by N5"
)

array_metadata["dataType"] = dtype.name
array_metadata["dimensions"] = array_metadata["dimensions"][::-1]
array_metadata["blockSize"] = array_metadata["blockSize"][::-1]

if "fill_value" in array_metadata:
if array_metadata["fill_value"] != 0 and array_metadata["fill_value"] is not None:
if (
array_metadata["fill_value"] != 0
and array_metadata["fill_value"] is not None
):
raise ValueError(
f"""Received fill_value = {array_metadata['fill_value']},
but N5 only supports fill_value = 0"""
Expand All @@ -634,7 +655,9 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic

if "filters" in array_metadata:
if array_metadata["filters"] != [] and array_metadata["filters"] is not None:
raise ValueError("Received filters, but N5 storage does not support zarr filters")
raise ValueError(
"Received filters, but N5 storage does not support zarr filters"
)
del array_metadata["filters"]

assert "compression" in array_metadata
Expand Down Expand Up @@ -691,7 +714,9 @@ def attrs_to_zarr(attrs: Dict[str, Any]) -> Dict[str, Any]:
return attrs


def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
def compressor_config_to_n5(
compressor_config: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
if compressor_config is None:
return {"type": "raw"}
else:
Expand Down Expand Up @@ -751,7 +776,9 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict
return n5_config


def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
def compressor_config_to_zarr(
compressor_config: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
codec_id = compressor_config["type"]
zarr_config = {"id": codec_id}

Expand Down Expand Up @@ -779,7 +806,7 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic
zarr_config["filters"] = None

elif codec_id == "gzip":
if "useZlib" in compressor_config and compressor_config["useZlib"]:
if compressor_config.get("useZlib"):
zarr_config["id"] = "zlib"
zarr_config["level"] = compressor_config["level"]
else:
Expand Down Expand Up @@ -808,10 +835,16 @@ def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None):

if compressor:
if compressor_config is not None:
raise ValueError("Only one of compressor_config or compressor should be given.")
raise ValueError(
"Only one of compressor_config or compressor should be given."
)
compressor_config = compressor.get_config()

if compressor_config is None and compressor is None or compressor_config["id"] == "raw":
if (
compressor_config is None
and compressor is None
or compressor_config["id"] == "raw"
):
self.compressor_config = None
self._compressor = None
else:
Expand Down Expand Up @@ -884,7 +917,8 @@ def _create_header(chunk):
def _read_header(chunk):
num_dims = struct.unpack(">H", chunk[2:4])[0]
shape = tuple(
struct.unpack(">I", chunk[i : i + 4])[0] for i in range(4, num_dims * 4 + 4, 4)
struct.unpack(">I", chunk[i : i + 4])[0]
for i in range(4, num_dims * 4 + 4, 4)
)[::-1]

len_header = 4 + num_dims * 4
Expand All @@ -908,4 +942,4 @@ def _from_big_endian(self, data):
return a.astype(self.dtype)


register_codec(N5ChunkWrapper, N5ChunkWrapper.codec_id)
register_codec(N5ChunkWrapper, N5ChunkWrapper.codec_id)
6 changes: 4 additions & 2 deletions tests/test_n5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ def test_make_n5_chunk_wrapper():
dtype,
chunk_shape=chunk_shape,
compressor_config=codec.get_config(),
compressor=codec
compressor=codec,
)

wrapper_a = N5ChunkWrapper(dtype, chunk_shape=chunk_shape, compressor_config=codec.get_config())
wrapper_a = N5ChunkWrapper(
dtype, chunk_shape=chunk_shape, compressor_config=codec.get_config()
)
wrapper_b = N5ChunkWrapper(dtype, chunk_shape=chunk_shape, compressor=codec)
assert wrapper_a == wrapper_b

Expand Down

0 comments on commit 9b29fbf

Please sign in to comment.