From 3387a261b2e7b3a99e3ac529d287680a07590312 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 3 Sep 2025 09:01:30 -0700 Subject: [PATCH 1/2] Allow tests to pass locally on Mac move env vars into the tests, out of the env setup. xfail tests without codec dependencies (I think this is a PyPI vs conda diff?) --- .github/workflows/ci.yaml | 2 - .github/workflows/upstream-dev-ci.yaml | 2 - conftest.py | 6 ++ xarray/tests/test_backends.py | 111 +++++++++++++++++++++++-- 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce9c47dedd..58193c1c390 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,8 +37,6 @@ jobs: runs-on: ${{ matrix.os }} needs: detect-ci-trigger if: needs.detect-ci-trigger.outputs.triggered == 'false' - env: - ZARR_V3_EXPERIMENTAL_API: 1 defaults: run: shell: bash -l {0} diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 0b3538c5cd8..aa83727ff99 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -40,8 +40,6 @@ jobs: name: upstream-dev runs-on: ubuntu-latest needs: detect-ci-trigger - env: - ZARR_V3_EXPERIMENTAL_API: 1 if: | always() && ( diff --git a/conftest.py b/conftest.py index 200696431ea..ed197c2bae4 100644 --- a/conftest.py +++ b/conftest.py @@ -41,6 +41,12 @@ def pytest_collection_modifyitems(items): item.add_marker(pytest.mark.mypy) +@pytest.fixture(autouse=True) +def set_zarr_v3_api(monkeypatch): + """Set ZARR_V3_EXPERIMENTAL_API environment variable for all tests.""" + monkeypatch.setenv("ZARR_V3_EXPERIMENTAL_API", "1") + + @pytest.fixture(autouse=True) def add_standard_imports(doctest_namespace, tmpdir): import numpy as np diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 95c53786f86..96639c08863 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -154,6 +154,66 @@ def skip_if_zarr_format_2(reason: str): ON_WINDOWS = sys.platform == "win32" default_value = object() + + +def _check_compression_codec_available(codec: str | None) -> bool: + """Check if a compression codec is available in the netCDF4 library. + + Parameters + ---------- + codec : str or None + The compression codec name (e.g., 'zstd', 'blosc_lz', etc.) + + Returns + ------- + bool + True if the codec is available, False otherwise. + """ + if codec is None or codec in ("zlib", "szip"): + # These are standard and should be available + return True + + if not has_netCDF4: + return False + + try: + import os + import tempfile + + import netCDF4 + + # Try to create a file with the compression to test availability + with tempfile.NamedTemporaryFile(suffix=".nc", delete=False) as tmp: + tmp_path = tmp.name + + try: + nc = netCDF4.Dataset(tmp_path, "w", format="NETCDF4") + nc.createDimension("x", 10) + + # Attempt to create a variable with the compression + if codec and codec.startswith("blosc"): + nc.createVariable( + "test", "f4", ("x",), compression=codec, blosc_shuffle=1 + ) + else: + nc.createVariable("test", "f4", ("x",), compression=codec) + + nc.close() + os.unlink(tmp_path) + return True + except (RuntimeError, netCDF4.NetCDF4MissingFeatureException): + # Codec not available + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + return False + except Exception: + # Any other error, assume codec is not available + return False + + dask_array_type = array_type("dask") if TYPE_CHECKING: @@ -2097,12 +2157,48 @@ def test_setncattr_string(self) -> None: None, "zlib", "szip", - "zstd", - "blosc_lz", - "blosc_lz4", - "blosc_lz4hc", - "blosc_zlib", - "blosc_zstd", + pytest.param( + "zstd", + marks=pytest.mark.xfail( + not _check_compression_codec_available("zstd"), + reason="zstd codec not available in netCDF4 installation", + ), + ), + pytest.param( + "blosc_lz", + marks=pytest.mark.xfail( + not _check_compression_codec_available("blosc_lz"), + reason="blosc_lz codec not available in netCDF4 installation", + ), + ), + pytest.param( + "blosc_lz4", + marks=pytest.mark.xfail( + not _check_compression_codec_available("blosc_lz4"), + reason="blosc_lz4 codec not available in netCDF4 installation", + ), + ), + pytest.param( + "blosc_lz4hc", + marks=pytest.mark.xfail( + not _check_compression_codec_available("blosc_lz4hc"), + reason="blosc_lz4hc codec not available in netCDF4 installation", + ), + ), + pytest.param( + "blosc_zlib", + marks=pytest.mark.xfail( + not _check_compression_codec_available("blosc_zlib"), + reason="blosc_zlib codec not available in netCDF4 installation", + ), + ), + pytest.param( + "blosc_zstd", + marks=pytest.mark.xfail( + not _check_compression_codec_available("blosc_zstd"), + reason="blosc_zstd codec not available in netCDF4 installation", + ), + ), ], ) @requires_netCDF4_1_6_2_or_above @@ -2379,7 +2475,8 @@ def test_read_non_consolidated_warning(self) -> None: def test_non_existent_store(self) -> None: with pytest.raises( - FileNotFoundError, match="(No such file or directory|Unable to find group)" + FileNotFoundError, + match="(No such file or directory|Unable to find group|does not exist)", ): xr.open_zarr(f"{uuid.uuid4()}") From 15263a6c347bc02dad5d2bed2a5f7467c557d34b Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 3 Sep 2025 09:07:31 -0700 Subject: [PATCH 2/2] Fix ruff linting issue: use contextlib.suppress instead of try-except-pass --- xarray/tests/test_backends.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index df2e188816e..c4292efaf6e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -218,10 +218,8 @@ def _check_compression_codec_available(codec: str | None) -> bool: except (RuntimeError, netCDF4.NetCDF4MissingFeatureException): # Codec not available if os.path.exists(tmp_path): - try: + with contextlib.suppress(OSError): os.unlink(tmp_path) - except OSError: - pass return False except Exception: # Any other error, assume codec is not available