diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e58c8f9dc9..971fc415af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,11 +129,10 @@ jobs: pip install hatch - name: Set Up Hatch Env run: | - hatch env create docs - hatch env run -e docs list-env + hatch run doctest:pip list - name: Run Tests run: | - hatch env run --env docs check + hatch run doctest:test test-complete: name: Test complete diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index ea48f8f622..0fea19a3f5 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -25,6 +25,7 @@ print(group) ```python exec="true" session="storage" source="above" result="ansi" # Implicitly create a read-only FsspecStore +# Note: requires s3fs to be installed group = zarr.open_group( store='s3://noaa-nwm-retro-v2-zarr-pds', mode='r', @@ -59,6 +60,7 @@ print(group) - an FSSpec URI string, indicating a [remote store](#remote-store) location: ```python exec="true" session="storage" source="above" result="ansi" + # Note: requires s3fs to be installed group = zarr.open_group( store='s3://noaa-nwm-retro-v2-zarr-pds', mode='r', @@ -125,6 +127,7 @@ that implements the [AbstractFileSystem](https://filesystem-spec.readthedocs.io/ API. `storage_options` can be used to configure the fsspec backend: ```python exec="true" session="storage" source="above" result="ansi" +# Note: requires s3fs to be installed store = zarr.storage.FsspecStore.from_url( 's3://noaa-nwm-retro-v2-zarr-pds', read_only=True, @@ -138,6 +141,7 @@ The type of filesystem (e.g. S3, https, etc..) is inferred from the scheme of th In case a specific filesystem is needed, one can explicitly create it. For example to create a S3 filesystem: ```python exec="true" session="storage" source="above" result="ansi" +# Note: requires s3fs to be installed import fsspec fs = fsspec.filesystem( 's3', anon=True, asynchronous=True, diff --git a/pyproject.toml b/pyproject.toml index 5733f42950..8095f87188 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -255,6 +255,18 @@ serve = "mkdocs serve" build = "mkdocs build" check = "mkdocs build --strict" readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" + +[tool.hatch.envs.doctest] +description = "Test environment for validating executable code blocks in documentation" +features = ['test', 'remote'] # Include remote dependencies for s3fs +dependencies = [ + "s3fs>=2023.10.0", + "pytest", + "pytest-examples", +] + +[tool.hatch.envs.doctest.scripts] +test = "pytest tests/test_docs.py -v" list-env = "pip list" [tool.ruff] @@ -396,7 +408,8 @@ addopts = [ ] filterwarnings = [ "error", - "ignore:Unclosed client session >> import zarr - >>> store = zarr.storage.MemoryStore() - >>> arr = await zarr.create_array( - >>> store=store, - >>> shape=(100,100), - >>> chunks=(10,10), - >>> dtype='i4', - >>> fill_value=0) - + ```python + import zarr + store = zarr.storage.MemoryStore() + arr = zarr.create_array( + store=store, + shape=(100,100), + chunks=(10,10), + dtype='i4', + fill_value=0) + # + ``` """ return Array( sync( @@ -1132,49 +1134,64 @@ def from_array( Examples -------- - Create an array from an existing Array:: - - >>> import zarr - >>> store = zarr.storage.MemoryStore() - >>> store2 = zarr.storage.LocalStore('example.zarr') - >>> arr = zarr.create_array( - >>> store=store, - >>> shape=(100,100), - >>> chunks=(10,10), - >>> dtype='int32', - >>> fill_value=0) - >>> arr2 = zarr.from_array(store2, data=arr) - - - Create an array from an existing NumPy array:: - - >>> import numpy as np - >>> arr3 = zarr.from_array( - zarr.storage.MemoryStore(), - >>> data=np.arange(10000, dtype='i4').reshape(100, 100), - >>> ) - - - Create an array from any array-like object:: - - >>> arr4 = zarr.from_array( - >>> zarr.storage.MemoryStore(), - >>> data=[[1, 2], [3, 4]], - >>> ) - - >>> arr4[...] - array([[1, 2],[3, 4]]) - - Create an array from an existing Array without copying the data:: - - >>> arr5 = zarr.from_array( - >>> zarr.storage.MemoryStore(), - >>> data=arr4, - >>> write_data=False, - >>> ) - - >>> arr5[...] - array([[0, 0],[0, 0]]) + Create an array from an existing Array: + + ```python + import zarr + store = zarr.storage.MemoryStore() + store2 = zarr.storage.LocalStore('example_from_array.zarr') + arr = zarr.create_array( + store=store, + shape=(100,100), + chunks=(10,10), + dtype='int32', + fill_value=0) + arr2 = zarr.from_array(store2, data=arr, overwrite=True) + # + ``` + + Create an array from an existing NumPy array: + + ```python + import zarr + import numpy as np + arr3 = zarr.from_array( + zarr.storage.MemoryStore(), + data=np.arange(10000, dtype='i4').reshape(100, 100), + ) + # + ``` + + Create an array from any array-like object: + + ```python + import zarr + arr4 = zarr.from_array( + zarr.storage.MemoryStore(), + data=[[1, 2], [3, 4]], + ) + # + arr4[...] + # array([[1, 2],[3, 4]]) + ``` + + Create an array from an existing Array without copying the data: + + ```python + import zarr + arr4 = zarr.from_array( + zarr.storage.MemoryStore(), + data=[[1, 2], [3, 4]], + ) + arr5 = zarr.from_array( + zarr.storage.MemoryStore(), + data=arr4, + write_data=False, + ) + # + arr5[...] + # array([[0, 0],[0, 0]]) + ``` """ return Array( sync( diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py index 651682d317..4a3d88a84f 100644 --- a/src/zarr/codecs/numcodecs/_codecs.py +++ b/src/zarr/codecs/numcodecs/_codecs.py @@ -3,18 +3,21 @@ These codecs were previously defined in [numcodecs][], and have now been moved to `zarr`. ->>> import numpy as np ->>> import zarr ->>> import zarr.codecs.numcodecs as numcodecs ->>> ->>> array = zarr.create_array( -... store="data.zarr", -... shape=(1024, 1024), -... chunks=(64, 64), -... dtype="uint32", -... filters=[numcodecs.Delta(dtype="uint32")], -... compressors=[numcodecs.BZ2(level=5)]) ->>> array[:] = np.arange(np.prod(array.shape), dtype=array.dtype).reshape(*array.shape) +```python +import numpy as np +import zarr +import zarr.codecs.numcodecs as numcodecs + +array = zarr.create_array( + store="data_numcodecs.zarr", + shape=(1024, 1024), + chunks=(64, 64), + dtype="uint32", + filters=[numcodecs.Delta(dtype="uint32")], + compressors=[numcodecs.BZ2(level=5)], + overwrite=True) +array[:] = np.arange(np.prod(array.shape), dtype=array.dtype).reshape(*array.shape) +``` !!! note Please note that the codecs in [zarr.codecs.numcodecs][] are not part of the Zarr version diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 59ca8f5929..42d6201ba9 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -982,10 +982,24 @@ async def open( Examples -------- - >>> import zarr - >>> store = zarr.storage.MemoryStore() - >>> async_arr = await AsyncArray.open(store) # doctest: +ELLIPSIS - + ```python + import asyncio + import zarr + from zarr.core.array import AsyncArray + + async def example(): + store = zarr.storage.MemoryStore() + # First create an array to open + await zarr.api.asynchronous.create_array( + store=store, shape=(100, 100), dtype="int32" + ) + # Now open it + async_arr = await AsyncArray.open(store) + return async_arr + + async_arr = asyncio.run(example()) + # + ``` """ store_path = await make_store_path(store) metadata_dict = await get_array_metadata(store_path, zarr_format=zarr_format) @@ -1300,12 +1314,23 @@ async def nchunks_initialized(self) -> int: Examples -------- - >>> arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(1,), shards=(2,)) - >>> await arr.nchunks_initialized() - 0 - >>> await arr.setitem(slice(5), 1) - >>> await arr.nchunks_initialized() - 6 + ```python + import asyncio + import zarr.api.asynchronous + + async def example(): + arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(1,)) + count = await arr.nchunks_initialized() + print(f"Initial: {count}") + #> Initial: 0 + await arr.setitem(slice(5), 1) + count = await arr.nchunks_initialized() + print(f"After write: {count}") + #> After write: 5 + return count + + result = asyncio.run(example()) + ``` """ if self.shards is None: chunks_per_shard = 1 @@ -1333,12 +1358,23 @@ async def _nshards_initialized(self) -> int: Examples -------- - >>> arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(2,)) - >>> await arr._nshards_initialized() - 0 - >>> await arr.setitem(slice(5), 1) - >>> await arr._nshards_initialized() - 3 + ```python + import asyncio + import zarr.api.asynchronous + + async def example(): + arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(2,)) + count = await arr._nshards_initialized() + print(f"Initial: {count}") + #> Initial: 0 + await arr.setitem(slice(5), 1) + count = await arr._nshards_initialized() + print(f"After write: {count}") + #> After write: 3 + return count + + result = asyncio.run(example()) + ``` """ return len(await _shards_initialized(self)) @@ -1566,18 +1602,25 @@ async def getitem( Examples -------- - >>> import zarr - >>> store = zarr.storage.MemoryStore() - >>> async_arr = await zarr.api.asynchronous.create_array( - ... store=store, - ... shape=(100,100), - ... chunks=(10,10), - ... dtype='i4', - ... fill_value=0) - - >>> await async_arr.getitem((0,1)) # doctest: +ELLIPSIS - array(0, dtype=int32) - + ```python + import asyncio + import zarr.api.asynchronous + + async def example(): + store = zarr.storage.MemoryStore() + async_arr = await zarr.api.asynchronous.create_array( + store=store, + shape=(100,100), + chunks=(10,10), + dtype='i4', + fill_value=0) + result = await async_arr.getitem((0,1)) + print(result) + #> 0 + return result + + value = asyncio.run(example()) + ``` """ if prototype is None: prototype = default_buffer_prototype() @@ -4014,7 +4057,7 @@ def blocks(self) -> BlockIndex: def resize(self, new_shape: ShapeLike) -> None: """ Change the shape of the array by growing or shrinking one or more - dimensions. + dimensions. This is an in-place operation that modifies the array. Parameters ---------- @@ -4032,20 +4075,20 @@ def resize(self, new_shape: ShapeLike) -> None: Examples -------- - >>> import zarr - >>> z = zarr.zeros(shape=(10000, 10000), - >>> chunk_shape=(1000, 1000), - >>> dtype="i4",) - >>> z.shape - (10000, 10000) - >>> z = z.resize(20000, 1000) - >>> z.shape - (20000, 1000) - >>> z2 = z.resize(50, 50) - >>> z.shape - (20000, 1000) - >>> z2.shape - (50, 50) + ```python + import zarr + z = zarr.zeros(shape=(10000, 10000), + chunk_shape=(1000, 1000), + dtype="int32",) + z.shape + #> (10000, 10000) + z.resize((20000, 1000)) + z.shape + #> (20000, 1000) + z.resize((50, 50)) + z.shape + #>(50, 50) + ``` """ sync(self._async_array.resize(new_shape)) diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index e000839436..7097385081 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -43,11 +43,11 @@ def put(self, d: dict[str, JSON]) -> None: Equivalent to the following pseudo-code, but performed atomically. ```python - >>> attrs = {"a": 1, "b": 2} - >>> attrs.clear() - >>> attrs.update({"a": 3", "c": 4}) - >>> attrs - {'a': 3, 'c': 4} + attrs = {"a": 1, "b": 2} + attrs.clear() + attrs.update({"a": "3", "c": 4}) + print(attrs) + #> {'a': '3', 'c': 4} ``` """ self._obj.metadata.attributes.clear() diff --git a/src/zarr/core/dtype/__init__.py b/src/zarr/core/dtype/__init__.py index bf09a7501e..f3077c32e5 100644 --- a/src/zarr/core/dtype/__init__.py +++ b/src/zarr/core/dtype/__init__.py @@ -213,14 +213,16 @@ def parse_data_type( Examples -------- - >>> from zarr.dtype import parse_data_type - >>> import numpy as np - >>> parse_data_type("int32", zarr_format=2) - Int32(endianness='little') - >>> parse_data_type(np.dtype('S10'), zarr_format=2) - NullTerminatedBytes(length=10) - >>> parse_data_type({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) - DateTime64(endianness='little', scale_factor=10, unit='s') + ```python + from zarr.dtype import parse_data_type + import numpy as np + parse_data_type("int32", zarr_format=2) + # Int32(endianness='little') + parse_data_type(np.dtype('S10'), zarr_format=2) + # NullTerminatedBytes(length=10) + parse_data_type({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) + # DateTime64(endianness='little', scale_factor=10, unit='s') + ``` """ return parse_dtype(dtype_spec, zarr_format=zarr_format) @@ -251,14 +253,16 @@ def parse_dtype( Examples -------- - >>> from zarr.dtype import parse_dtype - >>> import numpy as np - >>> parse_dtype("int32", zarr_format=2) - Int32(endianness='little') - >>> parse_dtype(np.dtype('S10'), zarr_format=2) - NullTerminatedBytes(length=10) - >>> parse_dtype({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) - DateTime64(endianness='little', scale_factor=10, unit='s') + ```python + from zarr.dtype import parse_dtype + import numpy as np + parse_dtype("int32", zarr_format=2) + # Int32(endianness='little') + parse_dtype(np.dtype('S10'), zarr_format=2) + # NullTerminatedBytes(length=10) + parse_dtype({"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 10}}, zarr_format=3) + # DateTime64(endianness='little', scale_factor=10, unit='s') + ``` """ if isinstance(dtype_spec, ZDType): return dtype_spec diff --git a/src/zarr/core/dtype/npy/string.py b/src/zarr/core/dtype/npy/string.py index ee8cc71aaf..41d3a60078 100644 --- a/src/zarr/core/dtype/npy/string.py +++ b/src/zarr/core/dtype/npy/string.py @@ -99,6 +99,7 @@ class FixedLengthUTF32JSON_V3(NamedConfig[Literal["fixed_length_utf32"], LengthB "name": "fixed_length_utf32", "configuration": { "length_bytes": 12 + } } ``` """ diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index 7aa546ea9c..8bedee07ef 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -74,7 +74,7 @@ class StructuredJSON_V3( "name": "structured", "configuration": { "fields": [ - ["f0", "int32], + ["f0", "int32"], ["f1", "float64"], ] } diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 71d2b52194..492211d097 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -292,21 +292,24 @@ def flattened_metadata(self) -> dict[str, ArrayV2Metadata | ArrayV3Metadata | Gr Examples -------- - >>> cm = ConsolidatedMetadata( - ... metadata={ - ... "group-0": GroupMetadata( - ... consolidated_metadata=ConsolidatedMetadata( - ... { - ... "group-0-0": GroupMetadata(), - ... } - ... ) - ... ), - ... "group-1": GroupMetadata(), - ... } - ... ) - {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), - 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), - 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group')} + ```python + from zarr.core.group import ConsolidatedMetadata, GroupMetadata + cm = ConsolidatedMetadata( + metadata={ + "group-0": GroupMetadata( + consolidated_metadata=ConsolidatedMetadata( + { + "group-0-0": GroupMetadata(), + } + ) + ), + "group-1": GroupMetadata(), + } + ) + # {'group-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), + # 'group-0/group-0-0': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group'), + # 'group-1': GroupMetadata(attributes={}, zarr_format=3, consolidated_metadata=None, node_type='group')} + ``` """ metadata = {} @@ -1894,16 +1897,19 @@ def __getitem__(self, path: str) -> Array | Group: Examples -------- - >>> import zarr - >>> group = Group.from_store(zarr.storage.MemoryStore() - >>> group.create_array(name="subarray", shape=(10,), chunks=(10,)) - >>> group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,)) - >>> group["subarray"] - - >>> group["subgroup"] - - >>> group["subgroup"]["subarray"] - + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup").create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group["subarray"] + # + group["subgroup"] + # + group["subgroup"]["subarray"] + # + ``` """ obj = self._sync(self._async_group.getitem(path)) @@ -1929,15 +1935,19 @@ def get(self, path: str, default: DefaultT | None = None) -> Array | Group | Def Examples -------- - >>> import zarr - >>> group = Group.from_store(zarr.storage.MemoryStore() - >>> group.create_array(name="subarray", shape=(10,), chunks=(10,)) - >>> group.create_group(name="subgroup") - >>> group.get("subarray") - - >>> group.get("subgroup") - - >>> group.get("nonexistent", None) + ```python + import zarr + from zarr.core.group import Group + group = Group.from_store(zarr.storage.MemoryStore()) + group.create_array(name="subarray", shape=(10,), chunks=(10,), dtype="float64") + group.create_group(name="subgroup") + group.get("subarray") + # + group.get("subgroup") + # + group.get("nonexistent", None) + # None + ``` """ try: diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 243096b029..c357ca7ccc 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -111,17 +111,20 @@ def _iter_grid( Examples -------- - >>> tuple(iter_grid((1,))) - ((0,),) + ```python + from zarr.core.indexing import _iter_grid + tuple(_iter_grid((1,))) + # ((0,),) - >>> tuple(iter_grid((2,3))) - ((0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)) + tuple(_iter_grid((2,3))) + # ((0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2)) - >>> tuple(iter_grid((2,3), origin=(1,1))) - ((1, 1), (1, 2)) + tuple(_iter_grid((2,3), origin=(1,1))) + # ((1, 1), (1, 2)) - >>> tuple(iter_grid((2,3), origin=(0,0), selection_shape=(2,2))) - ((0, 0), (0, 1), (1, 0), (1, 1)) + tuple(_iter_grid((2,3), origin=(0,0), selection_shape=(2,2))) + # ((0, 0), (0, 1), (1, 0), (1, 1)) + ``` """ if origin is None: origin_parsed = (0,) * len(grid_shape) @@ -190,17 +193,20 @@ def _iter_regions( Examples -------- - >>> tuple(iter_regions((1,), (1,))) - ((slice(0, 1, 1),),) + ```python + from zarr.core.indexing import _iter_regions + tuple(_iter_regions((1,), (1,))) + # ((slice(0, 1, 1),),) - >>> tuple(iter_regions((2, 3), (1, 2))) - ((slice(0, 1, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(0, 2, 1))) + tuple(_iter_regions((2, 3), (1, 2))) + # ((slice(0, 1, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(0, 2, 1))) - >>> tuple(iter_regions((2,3), (1,2)), origin=(1,1)) - ((slice(1, 2, 1), slice(1, 3, 1)), (slice(2, 3, 1), slice(1, 3, 1))) + tuple(_iter_regions((2,3), (1,2), origin=(1,1))) + # ((slice(1, 2, 1), slice(1, 3, 1)), (slice(2, 3, 1), slice(1, 3, 1))) - >>> tuple(iter_regions((2,3), (1,2)), origin=(1,1), selection_shape=(2,2)) - ((slice(1, 2, 1), slice(1, 3, 1)), (slice(2, 3, 1), slice(1, 3, 1))) + tuple(_iter_regions((2,3), (1,2), origin=(0,0), selection_shape=(2,2))) + # ((slice(0, 1, 1), slice(0, 2, 1)), (slice(1, 2, 1), slice(0, 2, 1))) + ``` """ grid_shape = tuple(ceildiv(d, s) for d, s in zip(domain_shape, region_shape, strict=True)) for grid_position in _iter_grid( diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index ffb04e764d..fe435cc2b8 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -128,10 +128,6 @@ def sync( ) -> T: """ Make loop run coroutine until it returns. Runs in other thread - - Examples - -------- - >>> sync(async_function(), existing_loop) """ if loop is None: # NB: if the loop is not running *yet*, it is OK to submit work diff --git a/src/zarr/core/sync_group.py b/src/zarr/core/sync_group.py index 39d8a17992..2a416f555f 100644 --- a/src/zarr/core/sync_group.py +++ b/src/zarr/core/sync_group.py @@ -94,15 +94,17 @@ def create_hierarchy( Examples -------- - >>> from zarr import create_hierarchy - >>> from zarr.storage import MemoryStore - >>> from zarr.core.group import GroupMetadata - - >>> store = MemoryStore() - >>> nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} - >>> nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) - >>> print(nodes) + ```python + from zarr import create_hierarchy + from zarr.storage import MemoryStore + from zarr.core.group import GroupMetadata + + store = MemoryStore() + nodes = {'a': GroupMetadata(attributes={'name': 'leaf'})} + nodes_created = dict(create_hierarchy(store=store, nodes=nodes)) + print(nodes) # {'a': GroupMetadata(attributes={'name': 'leaf'}, zarr_format=3, consolidated_metadata=None, node_type='group')} + ``` """ coro = create_hierarchy_async(store=store, nodes=nodes, overwrite=overwrite) diff --git a/src/zarr/experimental/cache_store.py b/src/zarr/experimental/cache_store.py index d228cc6a12..3456c94320 100644 --- a/src/zarr/experimental/cache_store.py +++ b/src/zarr/experimental/cache_store.py @@ -44,23 +44,25 @@ class CacheStore(WrapperStore[Store]): Examples -------- - >>> import zarr - >>> from zarr.storage import MemoryStore - >>> from zarr.experimental.cache_store import CacheStore - >>> - >>> # Create a cached store - >>> source_store = MemoryStore() - >>> cache_store = MemoryStore() - >>> cached_store = CacheStore( - ... store=source_store, - ... cache_store=cache_store, - ... max_age_seconds=60, - ... max_size=1024*1024 - ... ) - >>> - >>> # Use it like any other store - >>> array = zarr.create(shape=(100,), store=cached_store) - >>> array[:] = 42 + ```python + import zarr + from zarr.storage import MemoryStore + from zarr.experimental.cache_store import CacheStore + + # Create a cached store + source_store = MemoryStore() + cache_store = MemoryStore() + cached_store = CacheStore( + store=source_store, + cache_store=cache_store, + max_age_seconds=60, + max_size=1024*1024 + ) + + # Use it like any other store + array = zarr.create(shape=(100,), store=cached_store) + array[:] = 42 + ``` """ @@ -109,7 +111,6 @@ def __init__( self.key_insert_times = {} else: self.key_insert_times = key_insert_times - self.cache_set_data = cache_set_data self._cache_order = OrderedDict() self._current_size = 0 diff --git a/src/zarr/registry.py b/src/zarr/registry.py index 092b4cafc0..a8dd2a1c6c 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -325,10 +325,12 @@ def get_numcodec(data: CodecJSON_V2[str]) -> Numcodec: Examples -------- - - >>> codec = get_codec({'id': 'zlib', 'level': 1}) - >>> codec - Zlib(level=1) + ```python + from zarr.registry import get_numcodec + codec = get_numcodec({'id': 'zlib', 'level': 1}) + codec + # Zlib(level=1) + ``` """ from numcodecs.registry import get_codec diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py index 9ecfe4c201..d762097cc3 100644 --- a/src/zarr/storage/_common.py +++ b/src/zarr/storage/_common.py @@ -429,12 +429,15 @@ def _is_fsspec_uri(uri: str) -> bool: Examples -------- - >>> _is_fsspec_uri("s3://bucket") - True - >>> _is_fsspec_uri("my-directory") - False - >>> _is_fsspec_uri("local://my-directory") - False + ```python + from zarr.storage._common import _is_fsspec_uri + _is_fsspec_uri("s3://bucket") + # True + _is_fsspec_uri("my-directory") + # False + _is_fsspec_uri("local://my-directory") + # False + ``` """ return "://" in uri or ("::" in uri and "local://" not in uri) diff --git a/src/zarr/storage/_utils.py b/src/zarr/storage/_utils.py index 145790278c..39c28d44c3 100644 --- a/src/zarr/storage/_utils.py +++ b/src/zarr/storage/_utils.py @@ -84,10 +84,13 @@ def _join_paths(paths: Iterable[str]) -> str: Examples -------- - >>> _join_paths(["", "a", "b"]) - 'a/b' - >>> _join_paths(["a", "b", "c"]) - 'a/b/c' + ```python + from zarr.storage._utils import _join_paths + _join_paths(["", "a", "b"]) + # 'a/b' + _join_paths(["a", "b", "c"]) + # 'a/b/c' + ``` """ return "/".join(filter(lambda v: v != "", paths)) @@ -116,10 +119,13 @@ def _relativize_path(*, path: str, prefix: str) -> str: Examples -------- - >>> _relativize_path(path="", prefix="a/b") - 'a/b' - >>> _relativize_path(path="a/b", prefix="a/b/c") - 'c' + ```python + from zarr.storage._utils import _relativize_path + _relativize_path(path="a/b", prefix="") + # 'a/b' + _relativize_path(path="a/b/c", prefix="a/b") + # 'c' + ``` """ if prefix == "": return path diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000000..d467e478e8 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,120 @@ +""" +Tests for executable code blocks in markdown documentation. + +This module uses pytest-examples to validate that all Python code examples +with exec="true" in the documentation execute successfully. +""" + +from __future__ import annotations + +from collections import defaultdict +from pathlib import Path + +import pytest + +pytest.importorskip("pytest_examples") +from pytest_examples import CodeExample, EvalExample, find_examples + +# Find all markdown files with executable code blocks +DOCS_ROOT = Path(__file__).parent.parent / "docs" +SOURCES_ROOT = Path(__file__).parent.parent / "src" / "zarr" + + +def find_markdown_files_with_exec() -> list[Path]: + """Find all markdown files containing exec="true" code blocks.""" + markdown_files = [] + + for md_file in DOCS_ROOT.rglob("*.md"): + try: + content = md_file.read_text(encoding="utf-8") + if 'exec="true"' in content: + markdown_files.append(md_file) + except Exception: + # Skip files that can't be read + continue + + return sorted(markdown_files) + + +def group_examples_by_session() -> list[tuple[str, str]]: + """ + Group examples by their session and file, maintaining order. + + Returns a list of session_key tuples where session_key is + (file_path, session_name). + """ + all_examples = list(find_examples(DOCS_ROOT)) + + # Group by file and session + sessions = defaultdict(list) + + for example in all_examples: + settings = example.prefix_settings() + if settings.get("exec") != "true": + continue + + # Use file path and session name as key + file_path = example.path + session_name = settings.get("session", "_default") + session_key = (str(file_path), session_name) + + sessions[session_key].append(example) + + # Return sorted list of session keys for consistent test ordering + return sorted(sessions.keys(), key=lambda x: (x[0], x[1])) + + +def name_example(path: str, session: str) -> str: + """Generate a readable name for a test case from file path and session.""" + return f"{Path(path).relative_to(DOCS_ROOT)}:{session}" + + +# Get all example sessions +@pytest.mark.parametrize( + "session_key", group_examples_by_session(), ids=lambda v: name_example(v[0], v[1]) +) +def test_documentation_examples( + session_key: tuple[str, str], + eval_example: EvalExample, +) -> None: + """ + Test that all exec="true" code examples in documentation execute successfully. + + This test groups examples by session (file + session name) and runs them + sequentially in the same execution context, allowing code to build on + previous examples. + + This test uses pytest-examples to: + - Find all code examples with exec="true" in markdown files + - Group them by session + - Execute them in order within the same context + - Verify no exceptions are raised + """ + file_path, session_name = session_key + + # Get examples for this session + all_examples = list(find_examples(DOCS_ROOT)) + examples = [] + for example in all_examples: + settings = example.prefix_settings() + if settings.get("exec") != "true": + continue + if str(example.path) == file_path and settings.get("session", "_default") == session_name: + examples.append(example) + + # Run all examples in this session sequentially, preserving state + module_globals: dict[str, object] = {} + for example in examples: + # TODO: uncomment this line when we are ready to fix output checks + # result = eval_example.run_print_check(example, module_globals=module_globals) + result = eval_example.run(example, module_globals=module_globals) + # Update globals with the results from this execution + module_globals.update(result) + + +@pytest.mark.parametrize("example", find_examples(str(SOURCES_ROOT)), ids=str) +def test_docstrings(example: CodeExample, eval_example: EvalExample) -> None: + """Test our docstring examples.""" + if example.path.name == "config.py" and "your.module" in example.source: + pytest.skip("Skip testing docstring example that assumes nonexistent module.") + eval_example.run_print_check(example)