From 998579a4964961698a9221936cc0da767a531c3b Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 3 Jul 2025 15:41:04 +0200 Subject: [PATCH 01/14] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7d8ff69..65d056ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, ] -license = "MIT" +license = {text = "MIT License"} license-files = [ "LICENSE.txt", "c-blosc/LICENSE.txt", From e79793bbfe60e3da18319f92ec484b90de1a3b5d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 20 Nov 2025 17:58:11 +0100 Subject: [PATCH 02/14] update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59d4c9ee..ba984970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, ] -license = {text = "MIT License"} +license = "MIT" license-files = [ "LICENSE.txt", "c-blosc/LICENSE.txt", From e4417fda75c3059324c794bd3912588e67b89317 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 20 Nov 2025 23:22:48 +0100 Subject: [PATCH 03/14] abstract over crc32c modules --- numcodecs/checksum32.py | 32 ++++++++++++++++++++++---------- pyproject.toml | 5 ++++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index 9ecf212e..a62cb1b0 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -1,8 +1,8 @@ import abc +import importlib.util import struct import zlib -from contextlib import suppress -from types import ModuleType +from collections.abc import Callable from typing import Literal import numpy as np @@ -12,9 +12,24 @@ from .compat import ensure_contiguous_ndarray, ndarray_copy from .jenkins import jenkins_lookup3 -_crc32c: ModuleType | None = None -with suppress(ImportError): - import google_crc32c as _crc32c # type: ignore[no-redef, unused-ignore] +crc32c_checksum: Callable[[Buffer, int], int] | None + +if importlib.util.find_spec("google_crc32c") is not None: + import google_crc32c + + def crc32c_checksum(data: Buffer, value: int = 0) -> int: + if value == 0: + return google_crc32c.value(data) + else: + return google_crc32c.extend(value, data) + +elif importlib.util.find_spec("crc32c") is not None: + import crc32c + + def crc32c_checksum(data: Buffer, value: int = 0) -> int: + return crc32c.crc32c(data, value) +else: + crc32c_checksum = None CHECKSUM_LOCATION = Literal['start', 'end'] @@ -167,7 +182,7 @@ def decode(self, buf, out=None): return memoryview(b[:-4]) -if _crc32c: +if crc32c_checksum is not None: class CRC32C(Checksum32): """Codec add a crc32c checksum to the buffer. @@ -183,7 +198,4 @@ class CRC32C(Checksum32): @staticmethod def checksum(data: Buffer, value: int = 0) -> int: - if value == 0: - return _crc32c.value(data) # type: ignore[union-attr] - else: - return _crc32c.extend(value, data) # type: ignore[union-attr] + return crc32c_checksum(data, value) # type: ignore[misc] diff --git a/pyproject.toml b/pyproject.toml index 5043738d..4a9fb86a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,10 @@ pcodec = [ "pcodec>=0.3,<0.4", ] crc32c = [ - "google-crc32c>=1.5", + "crc32c>=2.7", +] +google_crc32c = [ + "google-crc32c>=1.5" ] [tool.setuptools] From 214a07359871cca23248132c567a19474e728168 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:17:21 -0500 Subject: [PATCH 04/14] Fix dependency groups/optional-dependencies --- pyproject.toml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4a9fb86a..ef3d734b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,23 @@ Documentation = "https://numcodecs.readthedocs.io/" Homepage = "https://github.com/zarr-developers/numcodecs" [project.optional-dependencies] +msgpack = [ + "msgpack", +] +zfpy = [ + "zfpy>=1.0.0" +] +pcodec = [ + "pcodec>=0.3,<0.4", +] +crc32c = [ + "crc32c>=2.7", +] +google_crc32c = [ + "google-crc32c>=1.5" +] + +[dependency-groups] docs = [ "sphinx", "sphinx-issues", @@ -63,8 +80,6 @@ test_extras = [ "importlib_metadata", "crc32c", # TODO: remove once zarr-python does not depend on crc32c anymore ] - -[dependency-groups] test-zarr-312 = [ "pytest", "pytest-cov", @@ -84,22 +99,6 @@ test-zarr-main = [ "crc32c", ] -msgpack = [ - "msgpack", -] -zfpy = [ - "zfpy>=1.0.0" -] -pcodec = [ - "pcodec>=0.3,<0.4", -] -crc32c = [ - "crc32c>=2.7", -] -google_crc32c = [ - "google-crc32c>=1.5" -] - [tool.setuptools] package-dir = {"" = "."} packages = ["numcodecs", "numcodecs.tests"] From b840e366ee564c88b1c9def43f7c76ed49c8f42f Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:24:28 -0500 Subject: [PATCH 05/14] Add deprecation warning --- numcodecs/checksum32.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index a62cb1b0..3d1262c1 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -1,6 +1,7 @@ import abc import importlib.util import struct +import warnings import zlib from collections.abc import Callable from typing import Literal @@ -24,6 +25,12 @@ def crc32c_checksum(data: Buffer, value: int = 0) -> int: return google_crc32c.extend(value, data) elif importlib.util.find_spec("crc32c") is not None: + warnings.warn( + "crc32c usage is deprecated since numcodecs v0.16.4. " + "It is recommended to install google_crc32c instead.", + DeprecationWarning, + stacklevel=2, + ) import crc32c def crc32c_checksum(data: Buffer, value: int = 0) -> int: From 69882195b93f80d2b287d53cf94adb80c0c9314c Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:28:32 -0500 Subject: [PATCH 06/14] Put back test and docs optional dependencies --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ef3d734b..b1e865e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,6 @@ crc32c = [ google_crc32c = [ "google-crc32c>=1.5" ] - -[dependency-groups] docs = [ "sphinx", "sphinx-issues", @@ -76,6 +74,8 @@ test = [ "pytest-cov", "pyzstd" ] + +[dependency-groups] test_extras = [ "importlib_metadata", "crc32c", # TODO: remove once zarr-python does not depend on crc32c anymore From c75539509f02a1c221275e0a3abc4900d6879cbf Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:29:21 -0500 Subject: [PATCH 07/14] Keep test-extras --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b1e865e4..61be998e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,12 +74,12 @@ test = [ "pytest-cov", "pyzstd" ] - -[dependency-groups] test_extras = [ "importlib_metadata", "crc32c", # TODO: remove once zarr-python does not depend on crc32c anymore ] + +[dependency-groups] test-zarr-312 = [ "pytest", "pytest-cov", From 9adb5c422451e8c4f275f8db7b8dff6da64c6239 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:47:49 -0500 Subject: [PATCH 08/14] Update deps in tests --- .github/workflows/ci-i386.yml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 39a5a6b8..de073886 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -63,7 +63,7 @@ jobs: run: | export DISABLE_NUMCODECS_AVX2="" uv venv - uv pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy] + uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] shell: alpine.sh {0} - name: List installed packages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e965b6c..7752d825 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,7 +50,7 @@ jobs: - name: Install numcodecs run: | export DISABLE_NUMCODECS_AVX2="" - python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy] + python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] - name: List installed packages run: python -m pip list From a4f4e020f6c781d9d601e7840b91b82483135b12 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:18:41 -0500 Subject: [PATCH 09/14] Add env for local testing --- docs/contributing.rst | 4 ++++ pyproject.toml | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 5bfff518..166fa826 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -101,6 +101,10 @@ like the following:: $ source ~/pyenv/numcodecs-dev/bin/activate $ pip install -e .[docs,test,msgpack,zfpy] +You may need to initialize the submodule for c-blosc: + + $ git submodule update --init --recursive + To verify that your development environment is working, you can run the unit tests:: $ pytest -v diff --git a/pyproject.toml b/pyproject.toml index 61be998e..7b77933a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,16 +253,32 @@ conflicts = [ ] ] -[tool.pixi.project] +[tool.pixi.workspace] channels = ["conda-forge"] platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] [tool.pixi.dependencies] +python = "=3.13" clang = ">=19.1.7,<20" c-compiler = ">=1.9.0,<2" cxx-compiler = ">=1.9.0,<2" uv = "*" +[tool.pixi.feature.test.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","pcodec","zfpy"] } + +[tool.pixi.feature.test-google-crc32c.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] } + +[tool.pixi.feature.test-crc32c.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["crc32c"] } + +[tool.pixi.environments] +default = { solve-group = "default" } +test = ["test"] +test-crc32c = ["test", "test-crc32c"] +test-google-crc32c = ["test", "test-google-crc32c"] + [tool.pixi.tasks] ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze" ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze" @@ -270,3 +286,6 @@ ls-deps-main = "uv run --group test-zarr-main uv pip freeze" test-zarr-312 = "uv run --group test-zarr-312 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" test-zarr-313 = "uv run --group test-zarr-313 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" test-zarr-main = "uv run --group test-zarr-main pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" + +[tool.pixi.feature.test.tasks] +run-tests = "pytest -v" \ No newline at end of file From 19653948e37961479a844338cb95ca588976dcec Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:18:54 -0500 Subject: [PATCH 10/14] Only emit warning once --- numcodecs/checksum32.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index 3d1262c1..9e792656 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -15,6 +15,8 @@ crc32c_checksum: Callable[[Buffer, int], int] | None +warnings.filterwarnings('once', message='crc32c usage is deprecated.*', category=DeprecationWarning) + if importlib.util.find_spec("google_crc32c") is not None: import google_crc32c From b43f161b06936759084bc044a6caff7615347a4d Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:19:28 -0500 Subject: [PATCH 11/14] Add test --- numcodecs/tests/test_checksum32.py | 40 +++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py index e97954cf..06b0a1a8 100644 --- a/numcodecs/tests/test_checksum32.py +++ b/numcodecs/tests/test_checksum32.py @@ -1,3 +1,4 @@ +import importlib import itertools from contextlib import suppress @@ -15,6 +16,9 @@ ) has_crc32c = False +has_google_crc32c = importlib.util.find_spec("google_crc32c") is not None +has_legacy_crc32c = importlib.util.find_spec("crc32c") is not None + with suppress(ImportError): from numcodecs.checksum32 import CRC32C @@ -104,7 +108,7 @@ def test_err_location(): with pytest.raises(ValueError): Adler32(location="foo") if not has_crc32c: - pytest.skip("Needs `crc32c` installed") + pytest.skip("Needs `crc32c` or `google_crc32c` installed") with pytest.raises(ValueError): CRC32C(location="foo") @@ -118,11 +122,15 @@ def test_err_location(): "Adler32(location='end')", pytest.param( "CRC32C(location='start')", - marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"), + marks=pytest.mark.skipif( + not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed" + ), ), pytest.param( "CRC32C(location='end')", - marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"), + marks=pytest.mark.skipif( + not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed" + ), ), ], ) @@ -141,7 +149,7 @@ def test_backwards_compatibility(codec_id, codec_instance): check_backwards_compatibility(codec_id, arrays, [codec_instance]) -@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed") +@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed") def test_backwards_compatibility_crc32c(): check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()]) @@ -164,14 +172,14 @@ def test_err_out_too_small(codec): codec.decode(codec.encode(arr), out) -@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed") +@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed") def test_crc32c_checksum(): arr = np.arange(0, 64, dtype="uint8") buf = CRC32C(location="end").encode(arr) assert np.frombuffer(buf, dtype=" Date: Thu, 20 Nov 2025 20:20:58 -0500 Subject: [PATCH 12/14] Lint --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b77933a..1c1bd11b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -288,4 +288,4 @@ test-zarr-313 = "uv run --group test-zarr-313 pytest numcodecs/tests/test_zarr3. test-zarr-main = "uv run --group test-zarr-main pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" [tool.pixi.feature.test.tasks] -run-tests = "pytest -v" \ No newline at end of file +run-tests = "pytest -v" From f58931059862e75f492068dcc1c540d15b0c1e3f Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:26:38 -0500 Subject: [PATCH 13/14] Don't install pcodec with python 3.14 --- .github/workflows/ci-i386.yml | 7 ++++++- .github/workflows/ci.yaml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index de073886..61f46b7c 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -63,7 +63,12 @@ jobs: run: | export DISABLE_NUMCODECS_AVX2="" uv venv - uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + # TODO: Remove this conditional when pcodec supports Python 3.14 + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + else + uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + fi shell: alpine.sh {0} - name: List installed packages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7752d825..2c6a5350 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,7 +50,12 @@ jobs: - name: Install numcodecs run: | export DISABLE_NUMCODECS_AVX2="" - python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + # TODO: Remove this conditional when pcodec supports Python 3.14 + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + else + python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + fi - name: List installed packages run: python -m pip list From 1d922db755613c80fb6f881514ea42d42742689e Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:40:33 -0500 Subject: [PATCH 14/14] Ignore slow google_crc32c warning --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c1bd11b..db2f3b61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,6 +154,7 @@ log_cli_level = "INFO" xfail_strict = true filterwarnings = [ "error", + "ignore:As the c extension couldn't be imported:RuntimeWarning", # Ignore warning about pure python google_crc32c (on Python 3.14) ] [tool.cibuildwheel] @@ -258,14 +259,14 @@ channels = ["conda-forge"] platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] [tool.pixi.dependencies] -python = "=3.13" +python = "=3.14" clang = ">=19.1.7,<20" c-compiler = ">=1.9.0,<2" cxx-compiler = ">=1.9.0,<2" uv = "*" [tool.pixi.feature.test.pypi-dependencies] -numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","pcodec","zfpy"] } +numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","zfpy"] } [tool.pixi.feature.test-google-crc32c.pypi-dependencies] numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] }