Parquet engine-core refactoring

dask/bytes/core.py

@@ -16,6 +16,7 @@
     infer_storage_options,
     build_name_function,
     update_storage_options,
+    stringify_path,
 )
 from .. import config
 from ..compatibility import unicode
@@ -327,7 +328,7 @@ def get_compression(urlpath, compression):
 def infer_options(urlpath):
     if hasattr(urlpath, "name"):
         # deal with pathlib.Path objects - must be local
-        urlpath = str(urlpath)
+        urlpath = stringify_path(urlpath)
         ispath = True
     else:
         ispath = False

dask/bytes/tests/test_s3.py

@@ -388,8 +388,8 @@ def test_read_text_passes_through_options():
 def test_parquet(s3, engine):
     dd = pytest.importorskip("dask.dataframe")
     lib = pytest.importorskip(engine)
-    if engine == "pyarrow" and LooseVersion(lib.__version__) == "0.13.0":
-        pytest.skip("pyarrow 0.13.0 not supported for parquet")
+    if engine == "pyarrow" and LooseVersion(lib.__version__) < "0.13.1":
+        pytest.skip("pyarrow < 0.13.01 not supported for parquet")
     import pandas as pd
     import numpy as np
 

dask/bytes/utils.py

@@ -37,6 +37,8 @@ def infer_storage_options(urlpath, inherit_storage_options=None):
     "host": "node", "port": 123, "path": "/mnt/datasets/test.csv",
     "url_query": "q=1", "extra": "value"}
     """
+    urlpath = stringify_path(urlpath)  # re, urllib don't support pathlib.Path objects
+
     # Handle Windows paths including disk name in this special case
     if re.match(r"^[a-zA-Z]:[\\/]", urlpath):
         return {"protocol": "file", "path": urlpath}