diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 52d07fb6f..298f874cf 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -8,10 +8,11 @@ import numpy as np import numpy.typing as npt -from zarr.abc.codec import Codec +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.array import Array, AsyncArray from zarr.buffer import NDArrayLike from zarr.chunk_key_encodings import ChunkKeyEncoding +from zarr.codecs.bytes import BytesCodec from zarr.common import JSON, ChunkCoords, MemoryOrder, OpenMode, ZarrFormat from zarr.group import AsyncGroup from zarr.metadata import ArrayV2Metadata, ArrayV3Metadata @@ -20,6 +21,8 @@ make_store_path, ) +default_pre_compressor = BytesCodec() + # TODO: this type could use some more thought, noqa to avoid "Variable "asynchronous.ArrayLike" is not valid as a type" ArrayLike = Union[AsyncArray | Array | npt.NDArray[Any]] # noqa PathLike = str @@ -532,7 +535,9 @@ async def create( *, # Note: this is a change from v2 chunks: ChunkCoords | None = None, # TODO: v2 allowed chunks=True dtype: npt.DTypeLike | None = None, - compressor: dict[str, JSON] | None = None, # TODO: default and type change + compressor: dict[str, JSON] | BytesBytesCodec | None = None, # TODO: default and type change + filters: Iterable[dict[str, JSON] | ArrayArrayCodec] = (), + pre_compressor: dict[str, JSON] | ArrayBytesCodec = default_pre_compressor, fill_value: Any = 0, # TODO: need type order: MemoryOrder | None = None, # TODO: default change store: str | StoreLike | None = None, @@ -540,7 +545,6 @@ async def create( overwrite: bool = False, path: PathLike | None = None, chunk_store: StoreLike | None = None, - filters: list[dict[str, JSON]] | None = None, # TODO: type has changed cache_metadata: bool | None = None, cache_attrs: bool | None = None, read_only: bool | None = None, @@ -559,7 +563,6 @@ async def create( | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, - codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, **kwargs: Any, ) -> AsyncArray: @@ -683,20 +686,27 @@ async def create( if path is not None: store_path = store_path / path + compressor_out: tuple[dict[str, JSON] | BytesBytesCodec, ...] + # normalize compressor to a tuple + if compressor is None: + compressor_out = () + else: + compressor_out = (compressor,) + return await AsyncArray.create( store_path, shape=shape, chunks=chunks, dtype=dtype, - compressor=compressor, + compressors=compressor_out, + filters=filters, + pre_compressor=pre_compressor, fill_value=fill_value, exists_ok=overwrite, # TODO: name change - filters=filters, dimension_separator=dimension_separator, zarr_format=zarr_format, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, - codecs=codecs, dimension_names=dimension_names, attributes=attributes, **kwargs, diff --git a/src/zarr/array.py b/src/zarr/array.py index 9ac1ce41e..a1dd3f20c 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -17,7 +17,7 @@ import numpy as np import numpy.typing as npt -from zarr.abc.codec import Codec +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.abc.store import set_or_delete from zarr.attributes import Attributes from zarr.buffer import BufferPrototype, NDArrayLike, NDBuffer, default_buffer_prototype @@ -113,14 +113,14 @@ async def create( | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, - codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, - filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + filters: Iterable[dict[str, JSON] | ArrayArrayCodec] = (), + pre_compressor: dict[str, JSON] | ArrayBytesCodec | None = None, + compressors: Iterable[dict[str, JSON] | BytesBytesCodec] = (), # runtime exists_ok: bool = False, ) -> AsyncArray: @@ -142,14 +142,6 @@ async def create( raise ValueError( "order cannot be used for arrays with version 3. Use a transpose codec instead." ) - if filters is not None: - raise ValueError( - "filters cannot be used for arrays with version 3. Use array-to-array codecs instead." - ) - if compressor is not None: - raise ValueError( - "compressor cannot be used for arrays with version 3. Use bytes-to-bytes codecs instead." - ) return await cls._create_v3( store_path, shape=shape, @@ -157,16 +149,14 @@ async def create( chunk_shape=chunk_shape, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, - codecs=codecs, + filters=filters, + pre_compressor=pre_compressor, + compressors=compressors, dimension_names=dimension_names, attributes=attributes, exists_ok=exists_ok, ) elif zarr_format == 2: - if codecs is not None: - raise ValueError( - "codecs cannot be used for arrays with version 2. Use filters and compressor instead." - ) if chunk_key_encoding is not None: raise ValueError( "chunk_key_encoding cannot be used for arrays with version 2. Use dimension_separator instead." @@ -182,7 +172,7 @@ async def create( fill_value=fill_value, order=order, filters=filters, - compressor=compressor, + compressor=pre_compressor, attributes=attributes, exists_ok=exists_ok, ) @@ -204,7 +194,9 @@ async def _create_v3( | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, - codecs: Iterable[Codec | dict[str, JSON]] | None = None, + pre_compressor: dict[str, JSON] | ArrayBytesCodec | None = None, + filters: Iterable[dict[str, JSON] | ArrayArrayCodec] = (), + compressors: Iterable[dict[str, JSON] | BytesBytesCodec] = (), dimension_names: Iterable[str] | None = None, attributes: dict[str, JSON] | None = None, exists_ok: bool = False, @@ -212,7 +204,15 @@ async def _create_v3( if not exists_ok: assert not await (store_path / ZARR_JSON).exists() - codecs = list(codecs) if codecs is not None else [BytesCodec()] + codecs: tuple[dict[str, JSON] | Codec, ...] + _pre_compressor: dict[str, JSON] | ArrayBytesCodec + + if pre_compressor is None: + _pre_compressor = BytesCodec() + else: + _pre_compressor = pre_compressor + + codecs = (*filters, _pre_compressor, *compressors) if fill_value is None: if dtype == np.dtype("bool"): @@ -258,8 +258,8 @@ async def _create_v2( dimension_separator: Literal[".", "/"] | None = None, fill_value: None | int | float = None, order: Literal["C", "F"] | None = None, - filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + filters: Iterable[dict[str, JSON] | Codec] = (), + compressor: dict[str, JSON] | ArrayBytesCodec | None = None, attributes: dict[str, JSON] | None = None, exists_ok: bool = False, ) -> AsyncArray: @@ -284,11 +284,7 @@ async def _create_v2( compressor=( numcodecs.get_codec(compressor).get_config() if compressor is not None else None ), - filters=( - [numcodecs.get_codec(filter).get_config() for filter in filters] - if filters is not None - else None - ), + filters=tuple(numcodecs.get_codec(filter).get_config() for filter in filters), attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path) @@ -596,14 +592,14 @@ def create( | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, - codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, - filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: dict[str, JSON] | ArrayBytesCodec | None = None, + filters: Iterable[dict[str, JSON] | ArrayArrayCodec] = (), + post_compressors: Iterable[dict[str, JSON] | BytesBytesCodec] = (), # runtime exists_ok: bool = False, ) -> Array: @@ -617,13 +613,13 @@ def create( fill_value=fill_value, chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, - codecs=codecs, dimension_names=dimension_names, chunks=chunks, dimension_separator=dimension_separator, order=order, + pre_compressor=compressor, filters=filters, - compressor=compressor, + compressors=post_compressors, exists_ok=exists_ok, ), ) diff --git a/src/zarr/group.py b/src/zarr/group.py index 4bb4b6b4d..ca7819df8 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -9,7 +9,7 @@ import numpy.typing as npt -from zarr.abc.codec import Codec +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec from zarr.abc.metadata import Metadata from zarr.abc.store import set_or_delete from zarr.array import Array, AsyncArray @@ -327,14 +327,14 @@ async def create_array( | tuple[Literal["v2"], Literal[".", "/"]] | None ) = None, - codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only chunks: ChunkCoords | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, - filters: list[dict[str, JSON]] | None = None, - compressor: dict[str, JSON] | None = None, + compressor: dict[str, JSON] | ArrayBytesCodec | None = None, + filters: Iterable[dict[str, JSON] | ArrayArrayCodec] = (), + post_compressors: Iterable[dict[str, JSON] | BytesBytesCodec] = (), # runtime exists_ok: bool = False, ) -> AsyncArray: @@ -345,14 +345,14 @@ async def create_array( chunk_shape=chunk_shape, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, - codecs=codecs, + pre_compressor=compressor, + filters=filters, + compressors=post_compressors, dimension_names=dimension_names, attributes=attributes, chunks=chunks, dimension_separator=dimension_separator, order=order, - filters=filters, - compressor=compressor, exists_ok=exists_ok, zarr_format=self.metadata.zarr_format, ) diff --git a/src/zarr/metadata.py b/src/zarr/metadata.py index 8329bd920..72f3ef772 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/metadata.py @@ -5,7 +5,10 @@ from collections.abc import Iterable from dataclasses import dataclass, field, replace from enum import Enum -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Any, Literal import numpy as np import numpy.typing as npt @@ -282,7 +285,7 @@ def from_dict(cls, data: dict[str, JSON]) -> ArrayV3Metadata: # check that the zarr_format attribute is correct _ = parse_zarr_format_v3(data.pop("zarr_format")) # type: ignore[arg-type] # check that the node_type attribute is correct - _ = parse_node_type_array(data.pop("node_type")) # type: ignore[arg-type] + _ = parse_node_type_array(data.pop("node_type")) data["dimension_names"] = data.pop("dimension_names", None) @@ -330,7 +333,7 @@ def __init__( order: Literal["C", "F"], dimension_separator: Literal[".", "/"] = ".", compressor: dict[str, JSON] | None = None, - filters: list[dict[str, JSON]] | None = None, + filters: Iterable[dict[str, JSON]] | None = None, attributes: dict[str, JSON] | None = None, ): """ @@ -463,26 +466,28 @@ def parse_attributes(data: None | dict[str, JSON]) -> dict[str, JSON]: # that takes 2 arguments def parse_zarr_format_v3(data: Literal[3]) -> Literal[3]: if data == 3: - return data + return 3 raise ValueError(f"Invalid value. Expected 3. Got {data}.") # todo: move to its own module and drop _v2 suffix def parse_zarr_format_v2(data: Literal[2]) -> Literal[2]: if data == 2: - return data + return 2 raise ValueError(f"Invalid value. Expected 2. Got {data}.") -def parse_node_type_array(data: Literal["array"]) -> Literal["array"]: +def parse_node_type_array(data: Any) -> Literal["array"]: if data == "array": - return data + return "array" raise ValueError(f"Invalid value. Expected 'array'. Got {data}.") # todo: real validation -def parse_filters(data: list[dict[str, JSON]] | None) -> list[dict[str, JSON]] | None: - return data +def parse_filters(data: Any) -> tuple[dict[str, JSON]] | None: + if isinstance(data, Iterable): + result = tuple(data) + return cast(tuple[dict[str, JSON]], result) # todo: real validation diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index e814afef1..2cc1aad82 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -112,14 +112,9 @@ async def test_codecs_use_of_prototype(): chunk_shape=(5, 5), dtype=expect.dtype, fill_value=0, - codecs=[ - TransposeCodec(order=(1, 0)), - BytesCodec(), - BloscCodec(), - Crc32cCodec(), - GzipCodec(), - ZstdCodec(), - ], + filters=[TransposeCodec(order=(1, 0))], + pre_compressor=BytesCodec(), + compressors=(BloscCodec(), Crc32cCodec(), GzipCodec(), ZstdCodec()), ) expect[:] = np.arange(100).reshape(10, 10) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index 514294c4b..31394eb02 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -75,17 +75,15 @@ def test_sharding( chunk_shape=(64, 64, 64), dtype=sample_data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", sample_data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + compressor=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ + TransposeCodec(order=order_from_dim("F", sample_data.ndim)), + BytesCodec(), + BloscCodec(cname="lz4"), + ], + index_location=index_location, + ), ) a[:, :, :] = sample_data @@ -105,17 +103,15 @@ def test_sharding_partial( chunk_shape=(64, 64, 64), dtype=sample_data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", sample_data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + compressor=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ + TransposeCodec(order=order_from_dim("F", sample_data.ndim)), + BytesCodec(), + BloscCodec(cname="lz4"), + ], + index_location=index_location, + ), ) a[10:, 10:, 10:] = sample_data @@ -138,17 +134,15 @@ def test_sharding_partial_read( chunk_shape=(64, 64, 64), dtype=sample_data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", sample_data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + compressor=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ + TransposeCodec(order=order_from_dim("F", sample_data.ndim)), + BytesCodec(), + BloscCodec(cname="lz4"), + ], + index_location=index_location, + ), ) read_data = a[0:10, 0:10, 0:10] @@ -167,17 +161,15 @@ def test_sharding_partial_overwrite( chunk_shape=(64, 64, 64), dtype=data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - TransposeCodec(order=order_from_dim("F", data.ndim)), - BytesCodec(), - BloscCodec(cname="lz4"), - ], - index_location=index_location, - ) - ], + compressor=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ + TransposeCodec(order=order_from_dim("F", data.ndim)), + BytesCodec(), + BloscCodec(cname="lz4"), + ], + index_location=index_location, + ), ) a[:10, :10, :10] = data @@ -211,15 +203,11 @@ def test_nested_sharding( chunk_shape=(64, 64, 64), dtype=sample_data.dtype, fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(32, 32, 32), - codecs=[ - ShardingCodec(chunk_shape=(16, 16, 16), index_location=inner_index_location) - ], - index_location=outer_index_location, - ) - ], + compressor=ShardingCodec( + chunk_shape=(32, 32, 32), + codecs=[ShardingCodec(chunk_shape=(16, 16, 16), index_location=inner_index_location)], + index_location=outer_index_location, + ), ) a[:, :, :] = sample_data @@ -243,17 +231,16 @@ async def test_order( with_sharding: bool, ): data = np.arange(0, 256, dtype="uint16").reshape((32, 8), order=input_order) - - codecs_: list[Codec] = ( - [ - ShardingCodec( - chunk_shape=(16, 8), - codecs=[TransposeCodec(order=order_from_dim(store_order, data.ndim)), BytesCodec()], - ) - ] - if with_sharding - else [TransposeCodec(order=order_from_dim(store_order, data.ndim)), BytesCodec()] - ) + compressor: Codec + filters: tuple[Codec] = () + if with_sharding: + compressor = ShardingCodec( + chunk_shape=(16, 8), + codecs=[TransposeCodec(order=order_from_dim(store_order, data.ndim)), BytesCodec()], + ) + else: + compressor = BytesCodec() + filters = (TransposeCodec(order=order_from_dim(store_order, data.ndim)),) with config.set({"array.order": runtime_write_order}): a = await AsyncArray.create( @@ -263,7 +250,8 @@ async def test_order( dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), - codecs=codecs_, + pre_compressor=compressor, + filters=filters, ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -311,7 +299,10 @@ def test_order_implicit( ): data = np.arange(0, 256, dtype="uint16").reshape((16, 16), order=input_order) - codecs_: list[Codec] | None = [ShardingCodec(chunk_shape=(8, 8))] if with_sharding else None + if with_sharding: + compressor = ShardingCodec(chunk_shape=(8, 8)) + else: + compressor = None with config.set({"array.order": runtime_write_order}): a = Array.create( @@ -320,7 +311,7 @@ def test_order_implicit( chunk_shape=(16, 16), dtype=data.dtype, fill_value=0, - codecs=codecs_, + compressor=compressor, ) a[:, :] = data @@ -353,16 +344,15 @@ async def test_transpose( ): data = np.arange(0, 256, dtype="uint16").reshape((1, 32, 8), order=input_order) - codecs_: list[Codec] = ( - [ - ShardingCodec( - chunk_shape=(1, 16, 8), - codecs=[TransposeCodec(order=(2, 1, 0)), BytesCodec()], - ) - ] - if with_sharding - else [TransposeCodec(order=(2, 1, 0)), BytesCodec()] - ) + if with_sharding: + compressor = ShardingCodec( + chunk_shape=(1, 16, 8), + codecs=[TransposeCodec(order=(2, 1, 0)), BytesCodec()], + ) + filters = () + else: + compressor = BytesCodec() + filters = (TransposeCodec(order=(2, 1, 0)),) with config.set({"array.order": runtime_write_order}): a = await AsyncArray.create( store / "transpose", @@ -371,7 +361,8 @@ async def test_transpose( dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), - codecs=codecs_, + pre_compressor=compressor, + filters=filters, ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -416,7 +407,8 @@ def test_transpose_non_self_inverse(store: Store, order): chunk_shape=data.shape, dtype=data.dtype, fill_value=0, - codecs=[TransposeCodec(order=order), BytesCodec()], + compressor=BytesCodec(), + filters=(TransposeCodec(order=order),), ) a[:, :] = data read_data = a[:, :] @@ -437,7 +429,8 @@ def test_transpose_invalid( dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), - codecs=[TransposeCodec(order=order), BytesCodec()], + compressor=BytesCodec(), + filters=(TransposeCodec(order=order),), ) @@ -460,16 +453,14 @@ def test_open_sharding(store: Store): chunk_shape=(16, 16), dtype="int32", fill_value=0, - codecs=[ - ShardingCodec( - chunk_shape=(8, 8), - codecs=[ - TransposeCodec(order=order_from_dim("F", 2)), - BytesCodec(), - BloscCodec(), - ], - ) - ], + compressor=ShardingCodec( + chunk_shape=(8, 8), + codecs=[ + TransposeCodec(order=order_from_dim("F", 2)), + BytesCodec(), + BloscCodec(), + ], + ), ) b = Array.open(store / "open_sharding") assert a.metadata == b.metadata @@ -603,15 +594,13 @@ def test_write_partial_sharded_chunks(store: Store): chunk_shape=(20, 20), dtype=data.dtype, fill_value=1, - codecs=[ - ShardingCodec( - chunk_shape=(10, 10), - codecs=[ - BytesCodec(), - BloscCodec(), - ], - ) - ], + compressor=ShardingCodec( + chunk_shape=(10, 10), + codecs=[ + BytesCodec(), + BloscCodec(), + ], + ), ) a[0:16, 0:16] = data assert np.array_equal(a[0:16, 0:16], data) @@ -640,7 +629,7 @@ async def test_delete_empty_shards(store: Store): chunk_shape=(8, 16), dtype="uint16", fill_value=1, - codecs=[ShardingCodec(chunk_shape=(8, 8))], + pre_compressor=ShardingCodec(chunk_shape=(8, 8)), ) await _AsyncArrayProxy(a)[:, :].set(np.zeros((16, 16))) await _AsyncArrayProxy(a)[8:, :].set(np.ones((8, 16))) @@ -699,7 +688,8 @@ async def test_zarr_compat_F(store: Store): dtype=data.dtype, chunk_key_encoding=("v2", "."), fill_value=1, - codecs=[TransposeCodec(order=order_from_dim("F", data.ndim)), BytesCodec()], + pre_compressor=BytesCodec(), + filters=(TransposeCodec(order=order_from_dim("F", data.ndim)),), ) z2 = zarr.v2.create( @@ -762,7 +752,8 @@ def test_gzip(store: Store): chunk_shape=(16, 16), dtype=data.dtype, fill_value=0, - codecs=[BytesCodec(), GzipCodec()], + compressor=BytesCodec(), + post_compressors=(GzipCodec(),), ) a[:, :] = data @@ -779,7 +770,8 @@ def test_zstd(store: Store, checksum: bool): chunk_shape=(16, 16), dtype=data.dtype, fill_value=0, - codecs=[BytesCodec(), ZstdCodec(level=0, checksum=checksum)], + compressor=BytesCodec(), + post_compressors=(ZstdCodec(level=0, checksum=checksum),), ) a[:, :] = data @@ -797,7 +789,7 @@ async def test_endian(store: Store, endian: Literal["big", "little"]): dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), - codecs=[BytesCodec(endian=endian)], + pre_compressor=BytesCodec(endian=endian), ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -832,7 +824,7 @@ async def test_endian_write( dtype="uint16", fill_value=0, chunk_key_encoding=("v2", "."), - codecs=[BytesCodec(endian=dtype_store_endian)], + pre_compressor=BytesCodec(endian=dtype_store_endian), ) await _AsyncArrayProxy(a)[:, :].set(data) @@ -868,35 +860,29 @@ def test_invalid_metadata(store: Store): chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - BytesCodec(endian="big"), - TransposeCodec(order=order_from_dim("F", 2)), - ], + compressor=BytesCodec(endian="big"), + post_compressors=(TransposeCodec(order=order_from_dim("F", 2)),), ) - with pytest.raises(TypeError): + with pytest.raises(ValueError): Array.create( - store / "invalid_order", + store / "invalid_missing_bytes_codec", shape=(16, 16), chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - BytesCodec(), - TransposeCodec(order="F"), - ], + compressor=TransposeCodec(order=order_from_dim("F", 2)), ) - with pytest.raises(ValueError): + with pytest.raises(TypeError): Array.create( - store / "invalid_missing_bytes_codec", + store / "invalid_order", shape=(16, 16), chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - TransposeCodec(order=order_from_dim("F", 2)), - ], + compressor=BytesCodec(), + post_compressors=(TransposeCodec(order="F"),), ) with pytest.raises(ValueError): @@ -906,9 +892,7 @@ def test_invalid_metadata(store: Store): chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - ShardingCodec(chunk_shape=(8,)), - ], + compressor=ShardingCodec(chunk_shape=(8,)), ) with pytest.raises(ValueError): Array.create( @@ -917,9 +901,7 @@ def test_invalid_metadata(store: Store): chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - ShardingCodec(chunk_shape=(8, 7)), - ], + compressor=ShardingCodec(chunk_shape=(8, 7)), ) with pytest.warns(UserWarning): @@ -929,10 +911,8 @@ def test_invalid_metadata(store: Store): chunk_shape=(16, 16), dtype=np.dtype("uint8"), fill_value=0, - codecs=[ - ShardingCodec(chunk_shape=(8, 8)), - GzipCodec(), - ], + compressor=ShardingCodec(chunk_shape=(8, 8)), + post_compressors=(GzipCodec(),), ) @@ -969,7 +949,8 @@ async def test_blosc_evolve(store: Store): chunk_shape=(16, 16), dtype="uint8", fill_value=0, - codecs=[BytesCodec(), BloscCodec()], + pre_compressor=BytesCodec(), + compressors=(BloscCodec(),), ) zarr_json = json.loads((await (store / "blosc_evolve_u1" / "zarr.json").get()).to_bytes()) @@ -983,7 +964,8 @@ async def test_blosc_evolve(store: Store): chunk_shape=(16, 16), dtype="uint16", fill_value=0, - codecs=[BytesCodec(), BloscCodec()], + pre_compressor=BytesCodec(), + compressors=(BloscCodec(),), ) zarr_json = json.loads((await (store / "blosc_evolve_u2" / "zarr.json").get()).to_bytes()) @@ -997,7 +979,7 @@ async def test_blosc_evolve(store: Store): chunk_shape=(16, 16), dtype="uint16", fill_value=0, - codecs=[ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()])], + pre_compressor=ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()]), ) zarr_json = json.loads((await (store / "sharding_blosc_evolve" / "zarr.json").get()).to_bytes())