[Requirements] Support PyArrow 3 (#1049)

mlrun/datastore/base.py

@@ -206,9 +206,15 @@ def reader(*args, **kwargs):
                     kwargs["storage_options"] = storage_options
                 return reader(url, **kwargs)
             else:
-                # If not dir, use fs.open() to avoid regression when pandas < 1.2 and does not
-                # support the storage_options parameter.
-                return reader(fs.open(url), **kwargs)
+
+                file = url
+                # Workaround for ARROW-12472 affecting pyarrow 3.x and 4.x.
+                if fs.protocol != "file":
+                    # If not dir, use fs.open() to avoid regression when pandas < 1.2 and does not
+                    # support the storage_options parameter.
+                    file = fs.open(url)
+
+                return reader(file, **kwargs)
 
         temp_file = tempfile.NamedTemporaryFile(delete=False)
         self.download(self._join(subpath), temp_file.name)

requirements.txt

@@ -25,7 +25,7 @@ numpy>=1.16.5, <1.20.0
 pandas~=1.2; python_version >= '3.7'
 pandas~=1.0; python_version < '3.7'
 # used as a the engine for parquet files by pandas
-pyarrow~=1.0
+pyarrow>=1,<4
 pyyaml~=5.1
 requests~=2.22
 sqlalchemy~=1.3

tests/test_requirements.py

@@ -113,6 +113,7 @@ def test_requirement_specifiers_convention():
         "boto3": {"~=1.9, <1.17.107"},
         "azure-storage-blob": {"~=12.0, <12.7.0"},
         "dask-ml": {"~=1.4,<1.9.0"},
+        "pyarrow": {">=1,<4"},
     }
 
     for (ignored_requirement_name, ignored_specifiers,) in ignored_invalid_map.items():