diff --git a/.github/workflows/test_wheel.yaml b/.github/workflows/test_wheel.yaml index 0ea20b8..723ca21 100644 --- a/.github/workflows/test_wheel.yaml +++ b/.github/workflows/test_wheel.yaml @@ -13,7 +13,6 @@ jobs: os: [ubuntu-22.04, windows-2019] architecture: ['x64'] linux_archs: ["native"] - numpy_version: ["numpy~=1.23", "numpy~=1.24", "numpy~=1.25", "numpy#latest"] include: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - os: windows-2019 @@ -22,7 +21,7 @@ jobs: linux_archs: native skip: "pp* *musl*" - name: Test wheel (${{ matrix.numpy_version }} + ${{ matrix.os }}) + name: Test wheel (${{ matrix.os }}) env: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_BEFORE_ALL: "pip install numpy cython" @@ -65,11 +64,10 @@ jobs: run: | pip install ./wheelhouse/*.whl - - name: Run Tests after installing numpy (${{matrix.numpy_version}}) + - name: Run Tests after installing shell: bash -l {0} run: | pip install pytest pytest-cov mv ./uavro ./uavro-src #in order to avoid conflicts between the uavro directory and the uavro installed module - pip install ${{matrix.numpy_version}} #installing a different numpy version than the one uavro wheel was compiled with pytest --verbose --cov=uavro-src #verifying that uavro still works diff --git a/ci/environment-py310.yml b/ci/environment-py310.yml index 152a013..34eb8a5 100644 --- a/ci/environment-py310.yml +++ b/ci/environment-py310.yml @@ -9,3 +9,6 @@ dependencies: - fastparquet - pytest - pytest-cov + - dask + - fsspec + - python-snappy diff --git a/ci/environment-py310win.yml b/ci/environment-py310win.yml index 152a013..34eb8a5 100644 --- a/ci/environment-py310win.yml +++ b/ci/environment-py310win.yml @@ -9,3 +9,6 @@ dependencies: - fastparquet - pytest - pytest-cov + - dask + - fsspec + - python-snappy diff --git a/ci/environment-py311.yml b/ci/environment-py311.yml index 904342d..8c90339 100644 --- a/ci/environment-py311.yml +++ b/ci/environment-py311.yml @@ -9,3 +9,6 @@ dependencies: - fastparquet - pytest - pytest-cov + - dask + - fsspec + - python-snappy diff --git a/ci/environment-py312.yml b/ci/environment-py312.yml index bad1310..4377381 100644 --- a/ci/environment-py312.yml +++ b/ci/environment-py312.yml @@ -9,3 +9,6 @@ dependencies: - fastparquet - pytest - pytest-cov + - dask + - fsspec + - python-snappy diff --git a/ci/environment-py39.yml b/ci/environment-py39.yml index a97197a..6101e3e 100644 --- a/ci/environment-py39.yml +++ b/ci/environment-py39.yml @@ -9,3 +9,6 @@ dependencies: - fastparquet - pytest - pytest-cov + - dask + - fsspec + - python-snappy diff --git a/uavro/core.py b/uavro/core.py index ded7e98..87b1c25 100644 --- a/uavro/core.py +++ b/uavro/core.py @@ -279,8 +279,9 @@ def dask_read_avro(urlpath, blocksize=100000000, storage_options=None): passed to backend file-system """ from dask import delayed, compute - from dask.bytes.core import open_files, read_bytes + from dask.bytes.core import read_bytes from dask.dataframe import from_delayed + from fsspec import open_files storage_options = storage_options or {} files = open_files(urlpath, **storage_options) diff --git a/uavro/tests/logical.avro b/uavro/tests/logical.avro index c35064c..8e5da69 100644 Binary files a/uavro/tests/logical.avro and b/uavro/tests/logical.avro differ diff --git a/uavro/tests/primitive.avro b/uavro/tests/primitive.avro index bc19826..4786a47 100644 Binary files a/uavro/tests/primitive.avro and b/uavro/tests/primitive.avro differ diff --git a/uavro/tests/util.py b/uavro/tests/util.py index f007aa9..c793116 100644 --- a/uavro/tests/util.py +++ b/uavro/tests/util.py @@ -17,9 +17,9 @@ def generate_test_data_primitive(fn): {"name": "d", "type": "long"}, {"name": "e", "type": "float"}, {"name": "f", "type": "double"}, - {"name": "g", "type": {'type': "fixed", 'size': 4}}, + {"name": "g", "type": {'type': "fixed", 'size': 4, 'name': 'g'}}, {"name": "h", "type": {'type': "enum", - "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"]}}, + "symbols": ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"], "name": "h"}}, ] } with open(fn, 'wb') as out: @@ -48,7 +48,8 @@ def generate_test_data_logical(fn): {"name": "e", "type": {'type': "int", 'logicalType': 'date'}}, {"name": "f", "type": {'type': "bytes", - 'logicalType': 'decimal', 'scale': 3}}, + 'logicalType': 'decimal', 'scale': 3, + 'precision': 38}}, ] } with open(fn, 'wb') as out: