Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add type hints to zarr.create #1536

Merged
merged 13 commits into from Dec 8, 2023
6 changes: 6 additions & 0 deletions docs/release.rst
Expand Up @@ -18,6 +18,12 @@ Release notes
Unreleased
----------

Enhancements
~~~~~~~~~~~~

* Added type hints to ``zarr.creation.create()``.
By :user:`David Stansby <dstansby>` :issue:`1536`.

Docs
~~~~

Expand Down
3 changes: 2 additions & 1 deletion zarr/_storage/store.py
Expand Up @@ -9,6 +9,7 @@
from zarr.meta import Metadata2, Metadata3
from zarr.util import normalize_storage_path
from zarr.context import Context
from zarr.types import ZARR_VERSION

# v2 store keys
array_meta_key = ".zarray"
Expand All @@ -19,7 +20,7 @@
meta_root = "meta/root/"
data_root = "data/root/"

DEFAULT_ZARR_VERSION = 2
DEFAULT_ZARR_VERSION: ZARR_VERSION = 2

v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"]

Expand Down
46 changes: 26 additions & 20 deletions zarr/creation.py
@@ -1,7 +1,10 @@
from typing import Optional
from collections.abc import MutableMapping
from typing import Optional, Tuple, Union, Sequence
from warnings import warn

import numpy as np
import numpy.typing as npt
from numcodecs.abc import Codec
from numcodecs.registry import codec_registry

from zarr._storage.store import DEFAULT_ZARR_VERSION
Expand All @@ -19,32 +22,35 @@
normalize_storage_path,
normalize_store_arg,
)
from zarr._storage.store import StorageTransformer
from zarr.sync import Synchronizer
from zarr.types import ZARR_VERSION, DIMENSION_SEPARATOR, MEMORY_ORDER, MetaArray, PathLike
from zarr.util import normalize_dimension_separator


def create(
shape,
chunks=True,
dtype=None,
shape: Union[int, Tuple[int, ...]],
chunks: bool = True,
dtype: Optional[npt.DTypeLike] = None,
d-v-b marked this conversation as resolved.
Show resolved Hide resolved
compressor="default",
fill_value: Optional[int] = 0,
order="C",
store=None,
synchronizer=None,
overwrite=False,
path=None,
chunk_store=None,
filters=None,
cache_metadata=True,
cache_attrs=True,
read_only=False,
object_codec=None,
dimension_separator=None,
write_empty_chunks=True,
order: MEMORY_ORDER = "C",
store: Optional[Union[str, MutableMapping]] = None,
synchronizer: Optional[Synchronizer] = None,
overwrite: bool = False,
path: Optional[PathLike] = None,
chunk_store: Optional[MutableMapping] = None,
filters: Optional[Sequence[Codec]] = None,
d-v-b marked this conversation as resolved.
Show resolved Hide resolved
cache_metadata: bool = True,
cache_attrs: bool = True,
read_only: bool = False,
object_codec: Optional[Codec] = None,
dimension_separator: Optional[DIMENSION_SEPARATOR] = None,
write_empty_chunks: bool = True,
*,
zarr_version=None,
meta_array=None,
storage_transformers=(),
zarr_version: Optional[ZARR_VERSION] = None,
meta_array: Optional[MetaArray] = None,
storage_transformers: Sequence[StorageTransformer] = (),
**kwargs,
):
"""Create an array.
Expand Down
2 changes: 1 addition & 1 deletion zarr/storage.py
Expand Up @@ -39,6 +39,7 @@
from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like
from numcodecs.registry import codec_registry
from zarr.context import Context
from zarr.types import PathLike as Path

from zarr.errors import (
MetadataError,
Expand Down Expand Up @@ -104,7 +105,6 @@
default_compressor = Zlib()


Path = Union[str, bytes, None]
# allow MutableMapping for backwards compatibility
StoreLike = Union[BaseStore, MutableMapping]

Expand Down
12 changes: 10 additions & 2 deletions zarr/sync.py
@@ -1,11 +1,19 @@
import os
from collections import defaultdict
from threading import Lock
from typing import Protocol

import fasteners


class ThreadSynchronizer:
class Synchronizer(Protocol):
"""Base class for synchronizers."""

def __getitem__(self, item):
...

Check warning on line 13 in zarr/sync.py

View check run for this annotation

Codecov / codecov/patch

zarr/sync.py#L13

Added line #L13 was not covered by tests
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thoughts welcome on how to make codecov happy with these lines.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't find anything obvious in https://docs.codecov.com/docs/ignoring-paths, seems like it only has per-file granularity. Since we know the decrease in coverage isn't meaningful, I'd be fine ignoring it here. In the future maybe we put protocols in types.py (either a module-local one, or a zarr-wide one) and tell codecov to ignore it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've updated the code coverage config to ignore lines that are "...".

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good idea!



class ThreadSynchronizer(Synchronizer):
"""Provides synchronization using thread locks."""

def __init__(self):
Expand All @@ -24,7 +32,7 @@
self.__init__()


class ProcessSynchronizer:
class ProcessSynchronizer(Synchronizer):
"""Provides synchronization using file locks via the
`fasteners <https://fasteners.readthedocs.io/en/latest/api/inter_process/>`_
package.
Expand Down
8 changes: 5 additions & 3 deletions zarr/tests/test_core.py
Expand Up @@ -3,7 +3,7 @@
import sys
import pickle
import shutil
from typing import Any, Literal, Optional, Tuple, Union
from typing import Any, Literal, Optional, Tuple, Union, Sequence
import unittest
from itertools import zip_longest
from tempfile import mkdtemp
Expand All @@ -26,6 +26,7 @@
VLenUTF8,
Zlib,
)
from numcodecs.abc import Codec
from numcodecs.compat import ensure_bytes, ensure_ndarray
from numcodecs.tests.common import greetings
from numpy.testing import assert_array_almost_equal, assert_array_equal
Expand Down Expand Up @@ -73,6 +74,7 @@
from zarr.tests.test_storage_v3 import DummyStorageTransfomer
from zarr.util import buffer_size
from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec, mktemp
from zarr.types import DIMENSION_SEPARATOR

# noinspection PyMethodMayBeStatic

Expand All @@ -82,8 +84,8 @@ class TestArray:
root = ""
path = ""
compressor = Zlib(level=1)
filters = None
dimension_separator: Literal["/", ".", None] = None
filters: Optional[Sequence[Codec]] = None
dimension_separator: Optional[DIMENSION_SEPARATOR] = None
cache_metadata = True
cache_attrs = True
partial_decompress: bool = False
Expand Down
13 changes: 13 additions & 0 deletions zarr/types.py
@@ -0,0 +1,13 @@
from typing import Literal, Protocol, Union

ZARR_VERSION = Literal[2, 3]
DIMENSION_SEPARATOR = Literal[".", "/"]
MEMORY_ORDER = Literal["C", "F"]


PathLike = Union[str, bytes, None]


class MetaArray(Protocol):
def __array_function__(self, func, types, args, kwargs):
...

Check warning on line 13 in zarr/types.py

View check run for this annotation

Codecov / codecov/patch

zarr/types.py#L13

Added line #L13 was not covered by tests
5 changes: 3 additions & 2 deletions zarr/util.py
Expand Up @@ -31,6 +31,7 @@
from numcodecs.ndarray_like import NDArrayLike
from numcodecs.registry import codec_registry
from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo
from zarr.types import DIMENSION_SEPARATOR

KeyType = TypeVar("KeyType")
ValueType = TypeVar("ValueType")
Expand Down Expand Up @@ -286,9 +287,9 @@ def normalize_order(order: str) -> str:
return order


def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]:
def normalize_dimension_separator(sep: Optional[str]) -> Optional[DIMENSION_SEPARATOR]:
if sep in (".", "/", None):
return sep
return cast(Optional[DIMENSION_SEPARATOR], sep)
else:
raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep)

Expand Down