Bump minimum Arrow version to 14.0.1 (#1304)

* Bump min arrow version to 14.0.1

* Remove pytest config diff

* Pin sqlalchemy<2 in 3.9 CI env

* Use explicit intake CSV driver to unblock v2 changes

continuous_integration/docker/conda.txt

@@ -13,7 +13,7 @@ tzlocal>=2.1
 fastapi>=0.92.0
 httpx>=0.24.1
 uvicorn>=0.13.4
-pyarrow>=6.0.2
+pyarrow>=14.0.1
 prompt_toolkit>=3.0.8
 pygments>=2.7.1
 scikit-learn>=1.0.0

continuous_integration/docker/main.dockerfile

@@ -26,7 +26,7 @@ RUN mamba install -y \
     "pygments>=2.7.1" \
     tabulate \
     # additional dependencies
-    "pyarrow>=6.0.2" \
+    "pyarrow>=14.0.1" \
     "scikit-learn>=1.0.0" \
     "intake>=0.6.0" \
     && conda clean -ay

continuous_integration/environment-3.10.yaml

@@ -11,14 +11,14 @@ dependencies:
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-- mlflow>=2.0
+- mlflow>=2.9
 - mock
-- numpy>=1.21.6
+- numpy>=1.22.4
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
 - psycopg2
-- pyarrow>=6.0.2
+- pyarrow>=14.0.1
 - pygments>=2.7.1
 - pyhive
 - pytest-cov

continuous_integration/environment-3.11.yaml

@@ -11,14 +11,14 @@ dependencies:
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-- mlflow>=2.0
+- mlflow>=2.9
 - mock
-- numpy>=1.21.6
+- numpy>=1.22.4
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
 - psycopg2
-- pyarrow>=6.0.2
+- pyarrow>=14.0.1
 - pygments>=2.7.1
 - pyhive
 - pytest-cov

continuous_integration/environment-3.12.yaml

@@ -12,14 +12,14 @@ dependencies:
 - lightgbm
 - maturin>=1.3,<1.4
 # TODO: add once mlflow 3.12 builds are available
-# - mlflow>=2.0
+# - mlflow>=2.9
 - mock
-- numpy>=1.21.6
+- numpy>=1.22.4
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
 - psycopg2
-- pyarrow>=6.0.2
+- pyarrow>=14.0.1
 - pygments>=2.7.1
 - pyhive
 - pytest-cov

continuous_integration/environment-3.9.yaml

@@ -11,14 +11,14 @@ dependencies:
 - jsonschema
 - lightgbm
 - maturin=1.3
-- mlflow=2.0
+- mlflow=2.9
 - mock
-- numpy=1.21.6
+- numpy=1.22.4
 - pandas=1.4.0
 - pre-commit
 - prompt_toolkit=3.0.8
 - psycopg2
-- pyarrow=6.0.2
+- pyarrow=14.0.1
 - pygments=2.7.1
 - pyhive
 - pytest-cov
@@ -28,7 +28,8 @@ dependencies:
 - python=3.9
 - scikit-learn=1.0.0
 - sphinx
-- sqlalchemy
+# TODO: remove this constraint when we require pandas>2
+- sqlalchemy<2
 - tpot>=0.12.0
 # FIXME: https://github.com/fugue-project/fugue/issues/526
 - triad<0.9.2

continuous_integration/gpuci/environment-3.10.yaml

@@ -17,14 +17,14 @@ dependencies:
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-- mlflow>=2.0
+- mlflow>=2.9
 - mock
-- numpy>=1.21.6
+- numpy>=1.22.4
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
 - psycopg2
-- pyarrow>=6.0.2
+- pyarrow>=14.0.1
 - pygments>=2.7.1
 - pyhive
 - pytest-cov

continuous_integration/gpuci/environment-3.9.yaml

@@ -17,14 +17,14 @@ dependencies:
 - jsonschema
 - lightgbm
 - maturin>=1.3,<1.4
-- mlflow>=2.0
+- mlflow>=2.9
 - mock
-- numpy>=1.21.6
+- numpy>=1.22.4
 - pandas>=1.4.0
 - pre-commit
 - prompt_toolkit>=3.0.8
 - psycopg2
-- pyarrow>=6.0.2
+- pyarrow>=14.0.1
 - pygments>=2.7.1
 - pyhive
 - pytest-cov

pyproject.toml

@@ -51,7 +51,7 @@ dev = [
     "pytest-cov>=2.10.1",
     "mock>=4.0.3",
     "sphinx>=3.2.1",
-    "pyarrow>=6.0.2",
+    "pyarrow>=14.0.1",
     "scikit-learn>=1.0.0",
     "intake>=0.6.0",
     "pre-commit",

tests/integration/test_intake.py

@@ -29,7 +29,7 @@ def intake_catalog_location():
         args:
             urlpath: "{{ CATALOG_DIR }}/data.csv"
         description: "Some Data"
-        driver: csv
+        driver: intake.source.csv.CSVSource
         """
         )