diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml index 39a5a6b8..61f46b7c 100644 --- a/.github/workflows/ci-i386.yml +++ b/.github/workflows/ci-i386.yml @@ -63,7 +63,12 @@ jobs: run: | export DISABLE_NUMCODECS_AVX2="" uv venv - uv pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy] + # TODO: Remove this conditional when pcodec supports Python 3.14 + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + else + uv pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + fi shell: alpine.sh {0} - name: List installed packages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e965b6c..2c6a5350 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,7 +50,12 @@ jobs: - name: Install numcodecs run: | export DISABLE_NUMCODECS_AVX2="" - python -m pip install -v -e .[test,test_extras,msgpack,crc32c,pcodec,zfpy] + # TODO: Remove this conditional when pcodec supports Python 3.14 + if [[ "${{ matrix.python-version }}" == "3.14" ]]; then + python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,zfpy] + else + python -m pip install -v -e .[test,test_extras,msgpack,google_crc32c,crc32c,pcodec,zfpy] + fi - name: List installed packages run: python -m pip list diff --git a/docs/contributing.rst b/docs/contributing.rst index 5bfff518..166fa826 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -101,6 +101,10 @@ like the following:: $ source ~/pyenv/numcodecs-dev/bin/activate $ pip install -e .[docs,test,msgpack,zfpy] +You may need to initialize the submodule for c-blosc: + + $ git submodule update --init --recursive + To verify that your development environment is working, you can run the unit tests:: $ pytest -v diff --git a/numcodecs/checksum32.py b/numcodecs/checksum32.py index 9ecf212e..9e792656 100644 --- a/numcodecs/checksum32.py +++ b/numcodecs/checksum32.py @@ -1,8 +1,9 @@ import abc +import importlib.util import struct +import warnings import zlib -from contextlib import suppress -from types import ModuleType +from collections.abc import Callable from typing import Literal import numpy as np @@ -12,9 +13,32 @@ from .compat import ensure_contiguous_ndarray, ndarray_copy from .jenkins import jenkins_lookup3 -_crc32c: ModuleType | None = None -with suppress(ImportError): - import google_crc32c as _crc32c # type: ignore[no-redef, unused-ignore] +crc32c_checksum: Callable[[Buffer, int], int] | None + +warnings.filterwarnings('once', message='crc32c usage is deprecated.*', category=DeprecationWarning) + +if importlib.util.find_spec("google_crc32c") is not None: + import google_crc32c + + def crc32c_checksum(data: Buffer, value: int = 0) -> int: + if value == 0: + return google_crc32c.value(data) + else: + return google_crc32c.extend(value, data) + +elif importlib.util.find_spec("crc32c") is not None: + warnings.warn( + "crc32c usage is deprecated since numcodecs v0.16.4. " + "It is recommended to install google_crc32c instead.", + DeprecationWarning, + stacklevel=2, + ) + import crc32c + + def crc32c_checksum(data: Buffer, value: int = 0) -> int: + return crc32c.crc32c(data, value) +else: + crc32c_checksum = None CHECKSUM_LOCATION = Literal['start', 'end'] @@ -167,7 +191,7 @@ def decode(self, buf, out=None): return memoryview(b[:-4]) -if _crc32c: +if crc32c_checksum is not None: class CRC32C(Checksum32): """Codec add a crc32c checksum to the buffer. @@ -183,7 +207,4 @@ class CRC32C(Checksum32): @staticmethod def checksum(data: Buffer, value: int = 0) -> int: - if value == 0: - return _crc32c.value(data) # type: ignore[union-attr] - else: - return _crc32c.extend(value, data) # type: ignore[union-attr] + return crc32c_checksum(data, value) # type: ignore[misc] diff --git a/numcodecs/tests/test_checksum32.py b/numcodecs/tests/test_checksum32.py index e97954cf..06b0a1a8 100644 --- a/numcodecs/tests/test_checksum32.py +++ b/numcodecs/tests/test_checksum32.py @@ -1,3 +1,4 @@ +import importlib import itertools from contextlib import suppress @@ -15,6 +16,9 @@ ) has_crc32c = False +has_google_crc32c = importlib.util.find_spec("google_crc32c") is not None +has_legacy_crc32c = importlib.util.find_spec("crc32c") is not None + with suppress(ImportError): from numcodecs.checksum32 import CRC32C @@ -104,7 +108,7 @@ def test_err_location(): with pytest.raises(ValueError): Adler32(location="foo") if not has_crc32c: - pytest.skip("Needs `crc32c` installed") + pytest.skip("Needs `crc32c` or `google_crc32c` installed") with pytest.raises(ValueError): CRC32C(location="foo") @@ -118,11 +122,15 @@ def test_err_location(): "Adler32(location='end')", pytest.param( "CRC32C(location='start')", - marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"), + marks=pytest.mark.skipif( + not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed" + ), ), pytest.param( "CRC32C(location='end')", - marks=pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed"), + marks=pytest.mark.skipif( + not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed" + ), ), ], ) @@ -141,7 +149,7 @@ def test_backwards_compatibility(codec_id, codec_instance): check_backwards_compatibility(codec_id, arrays, [codec_instance]) -@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed") +@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed") def test_backwards_compatibility_crc32c(): check_backwards_compatibility(CRC32C.codec_id, arrays, [CRC32C()]) @@ -164,14 +172,14 @@ def test_err_out_too_small(codec): codec.decode(codec.encode(arr), out) -@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` installed") +@pytest.mark.skipif(not has_crc32c, reason="Needs `crc32c` or `google_crc32c` installed") def test_crc32c_checksum(): arr = np.arange(0, 64, dtype="uint8") buf = CRC32C(location="end").encode(arr) assert np.frombuffer(buf, dtype="=1.0.0" +] +pcodec = [ + "pcodec>=0.3,<0.4", +] +crc32c = [ + "crc32c>=2.7", +] +google_crc32c = [ + "google-crc32c>=1.5" +] docs = [ "sphinx", "sphinx-issues", @@ -84,19 +99,6 @@ test-zarr-main = [ "crc32c", ] -msgpack = [ - "msgpack", -] -zfpy = [ - "zfpy>=1.0.0" -] -pcodec = [ - "pcodec>=0.3,<0.4", -] -crc32c = [ - "google-crc32c>=1.5", -] - [tool.setuptools] package-dir = {"" = "."} packages = ["numcodecs", "numcodecs.tests"] @@ -152,6 +154,7 @@ log_cli_level = "INFO" xfail_strict = true filterwarnings = [ "error", + "ignore:As the c extension couldn't be imported:RuntimeWarning", # Ignore warning about pure python google_crc32c (on Python 3.14) ] [tool.cibuildwheel] @@ -251,16 +254,32 @@ conflicts = [ ] ] -[tool.pixi.project] +[tool.pixi.workspace] channels = ["conda-forge"] platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] [tool.pixi.dependencies] +python = "=3.14" clang = ">=19.1.7,<20" c-compiler = ">=1.9.0,<2" cxx-compiler = ">=1.9.0,<2" uv = "*" +[tool.pixi.feature.test.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["test","test_extras","msgpack","zfpy"] } + +[tool.pixi.feature.test-google-crc32c.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["google_crc32c"] } + +[tool.pixi.feature.test-crc32c.pypi-dependencies] +numcodecs = { path = ".", editable = true, extras = ["crc32c"] } + +[tool.pixi.environments] +default = { solve-group = "default" } +test = ["test"] +test-crc32c = ["test", "test-crc32c"] +test-google-crc32c = ["test", "test-google-crc32c"] + [tool.pixi.tasks] ls-deps-312 = "uv run --group test-zarr-312 uv pip freeze" ls-deps-313 = "uv run --group test-zarr-313 uv pip freeze" @@ -268,3 +287,6 @@ ls-deps-main = "uv run --group test-zarr-main uv pip freeze" test-zarr-312 = "uv run --group test-zarr-312 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" test-zarr-313 = "uv run --group test-zarr-313 pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" test-zarr-main = "uv run --group test-zarr-main pytest numcodecs/tests/test_zarr3.py numcodecs/tests/test_zarr3_import.py" + +[tool.pixi.feature.test.tasks] +run-tests = "pytest -v"