From 1614d6ca5099f02002c0167b020f4f8c658df082 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 2 Aug 2019 15:54:30 -0500 Subject: [PATCH 1/4] Skip to_parquet tests --- dask/dataframe/io/tests/test_parquet.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py index 867a3b59f73..9e3f13e336a 100644 --- a/dask/dataframe/io/tests/test_parquet.py +++ b/dask/dataframe/io/tests/test_parquet.py @@ -733,7 +733,7 @@ def test_append_wo_index(tmpdir, engine): assert_eq(df.set_index("f"), ddf3) -def test_append_overlapping_divisions(tmpdir): +def test_append_overlapping_divisions(tmpdir, engine): """Test raising of error when divisions overlapping.""" tmp = str(tmpdir) df = pd.DataFrame( @@ -749,16 +749,16 @@ def test_append_overlapping_divisions(tmpdir): half = len(df) // 2 ddf1 = dd.from_pandas(df.iloc[:half], chunksize=100) ddf2 = dd.from_pandas(df.iloc[half - 10 :], chunksize=100) - ddf1.to_parquet(tmp) + ddf1.to_parquet(tmp, engine=engine) with pytest.raises(ValueError) as excinfo: - ddf2.to_parquet(tmp, append=True) + ddf2.to_parquet(tmp, engine=engine, append=True) assert "Appended divisions" in str(excinfo.value) - ddf2.to_parquet(tmp, append=True, ignore_divisions=True) + ddf2.to_parquet(tmp, engine=engine, append=True, ignore_divisions=True) -def test_append_different_columns(tmpdir): +def test_append_different_columns(tmpdir, engine): """Test raising of error when non equal columns.""" tmp = str(tmpdir) df1 = pd.DataFrame({"i32": np.arange(100, dtype=np.int32)}) @@ -769,14 +769,14 @@ def test_append_different_columns(tmpdir): ddf2 = dd.from_pandas(df2, chunksize=2) ddf3 = dd.from_pandas(df3, chunksize=2) - ddf1.to_parquet(tmp) + ddf1.to_parquet(tmp, engine=engine) with pytest.raises(ValueError) as excinfo: - ddf2.to_parquet(tmp, append=True) + ddf2.to_parquet(tmp, engine=engine, append=True) assert "Appended columns" in str(excinfo.value) with pytest.raises(ValueError) as excinfo: - ddf3.to_parquet(tmp, append=True) + ddf3.to_parquet(tmp, engine=engine, append=True) assert "Appended dtypes" in str(excinfo.value) From 2b20bfd804921b2438fdfc9c4f3250bfe0977d21 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 2 Aug 2019 15:56:09 -0500 Subject: [PATCH 2/4] Temporarily remove fastparquet / pyarrow from CI --- continuous_integration/travis/travis-35.yaml | 4 ++-- continuous_integration/travis/travis-36.yaml | 2 +- continuous_integration/travis/travis-37-dev.yaml | 4 ++-- continuous_integration/travis/travis-37.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/continuous_integration/travis/travis-35.yaml b/continuous_integration/travis/travis-35.yaml index 864638b528f..82fbe49d42f 100644 --- a/continuous_integration/travis/travis-35.yaml +++ b/continuous_integration/travis/travis-35.yaml @@ -12,14 +12,14 @@ dependencies: - pytest - pytest-xdist - moto - - fastparquet + # - fastparquet - h5py - pytables - zarr - fsspec - tiledb-py - sqlalchemy - - pyarrow + # - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-36.yaml b/continuous_integration/travis/travis-36.yaml index 29043ae3eca..23c1334d466 100644 --- a/continuous_integration/travis/travis-36.yaml +++ b/continuous_integration/travis/travis-36.yaml @@ -22,7 +22,7 @@ dependencies: - zarr - tiledb-py - sqlalchemy - - pyarrow + # - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-37-dev.yaml b/continuous_integration/travis/travis-37-dev.yaml index 0eac9fbe602..7f5bd38c00c 100644 --- a/continuous_integration/travis/travis-37-dev.yaml +++ b/continuous_integration/travis/travis-37-dev.yaml @@ -12,13 +12,13 @@ dependencies: - pytest - pytest-xdist - moto - - fastparquet + # - fastparquet - h5py - pytables - zarr - tiledb-py - sqlalchemy - - pyarrow + # - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-37.yaml b/continuous_integration/travis/travis-37.yaml index c34cf573091..52b004ecfe1 100644 --- a/continuous_integration/travis/travis-37.yaml +++ b/continuous_integration/travis/travis-37.yaml @@ -10,14 +10,14 @@ dependencies: - pytest - pytest-xdist - moto - - fastparquet + # - fastparquet - h5py - pytables - zarr - tiledb-py - fsspec - sqlalchemy - - pyarrow + # - pyarrow # other -- IO - bcolz - blosc From 629fbd6154a1f38122546a7c36a106d7b27b919c Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 2 Aug 2019 16:34:19 -0500 Subject: [PATCH 3/4] Add arrow.py to collect_ignore --- conftest.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/conftest.py b/conftest.py index d48c0f39d50..be66c1ef308 100644 --- a/conftest.py +++ b/conftest.py @@ -4,13 +4,16 @@ # - Non-required dependencies not being installed # - Imported doctests due to pulling the docstrings from other packages # (e.g. `numpy`). No need to run these doctests. -collect_ignore = ['dask/bytes/hdfs3.py', - 'dask/bytes/pyarrow.py', - 'dask/bytes/s3.py', - 'dask/array/ghost.py', - 'dask/array/fft.py', - 'dask/dataframe/io/io.py', - 'dask/dot.py'] +collect_ignore = [ + "dask/bytes/hdfs3.py", + "dask/bytes/pyarrow.py", + "dask/bytes/s3.py", + "dask/array/ghost.py", + "dask/array/fft.py", + "dask/dataframe/io/io.py", + "dask/dataframe/io/parquet/arrow.py", + "dask/dot.py", +] def pytest_addoption(parser): @@ -18,5 +21,5 @@ def pytest_addoption(parser): def pytest_runtest_setup(item): - if 'slow' in item.keywords and not item.config.getoption("--runslow"): + if "slow" in item.keywords and not item.config.getoption("--runslow"): pytest.skip("need --runslow option to run") From 36f6c434ded4f0a7cde68ef0993da99c3eb29dea Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 2 Aug 2019 17:13:01 -0500 Subject: [PATCH 4/4] Adds fastparquet and pyarrow back to CI environment --- continuous_integration/travis/travis-35.yaml | 4 ++-- continuous_integration/travis/travis-36.yaml | 2 +- continuous_integration/travis/travis-37-dev.yaml | 4 ++-- continuous_integration/travis/travis-37.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/continuous_integration/travis/travis-35.yaml b/continuous_integration/travis/travis-35.yaml index 82fbe49d42f..864638b528f 100644 --- a/continuous_integration/travis/travis-35.yaml +++ b/continuous_integration/travis/travis-35.yaml @@ -12,14 +12,14 @@ dependencies: - pytest - pytest-xdist - moto - # - fastparquet + - fastparquet - h5py - pytables - zarr - fsspec - tiledb-py - sqlalchemy - # - pyarrow + - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-36.yaml b/continuous_integration/travis/travis-36.yaml index 23c1334d466..29043ae3eca 100644 --- a/continuous_integration/travis/travis-36.yaml +++ b/continuous_integration/travis/travis-36.yaml @@ -22,7 +22,7 @@ dependencies: - zarr - tiledb-py - sqlalchemy - # - pyarrow + - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-37-dev.yaml b/continuous_integration/travis/travis-37-dev.yaml index 7f5bd38c00c..0eac9fbe602 100644 --- a/continuous_integration/travis/travis-37-dev.yaml +++ b/continuous_integration/travis/travis-37-dev.yaml @@ -12,13 +12,13 @@ dependencies: - pytest - pytest-xdist - moto - # - fastparquet + - fastparquet - h5py - pytables - zarr - tiledb-py - sqlalchemy - # - pyarrow + - pyarrow # other -- IO - bcolz - blosc diff --git a/continuous_integration/travis/travis-37.yaml b/continuous_integration/travis/travis-37.yaml index 52b004ecfe1..c34cf573091 100644 --- a/continuous_integration/travis/travis-37.yaml +++ b/continuous_integration/travis/travis-37.yaml @@ -10,14 +10,14 @@ dependencies: - pytest - pytest-xdist - moto - # - fastparquet + - fastparquet - h5py - pytables - zarr - tiledb-py - fsspec - sqlalchemy - # - pyarrow + - pyarrow # other -- IO - bcolz - blosc