explictly use http filesystem

ray-project · Jan 23, 2024 · 01cadf8 · 01cadf8
1 parent 804a4de
commit 01cadf8
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py
@@ -2431,7 +2431,10 @@ def from_huggingface(
         file_urls = HuggingFaceDatasource.list_parquet_urls_from_dataset(dataset)
         if len(file_urls) > 0:
             # If file urls are returned, the parquet files are available via API
-            return read_parquet(file_urls, parallelism=parallelism)
+            import fsspec.implementations.http
+
+            http = fsspec.implementations.http.HTTPFileSystem()
+            return read_parquet(file_urls, parallelism=parallelism, filesystem=http)
 
     if isinstance(dataset, datasets.IterableDataset):
         # For an IterableDataset, we can use a streaming implementation to read data.