Migrate all "list_remote_files_legacy" to "list_remote_files_fsspec"

tests/provider/dwd/observation/test_fileindex.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2018-2021, earthobservations developers.
 # Distributed under the MIT License. See LICENSE for more info.
 """ tests for file index creation """
+import aiohttp
 import pytest
 import requests
 
@@ -39,7 +40,9 @@ def test_file_index_creation():
         "climate/daily/kl/recent/tageswerte_KL_01048_akt.zip"
     ]
 
-    with pytest.raises(requests.exceptions.HTTPError):
+    with pytest.raises(
+        (requests.exceptions.HTTPError, aiohttp.client_exceptions.ClientResponseError)
+    ):
         create_file_index_for_climate_observations(
             DwdObservationDataset.CLIMATE_SUMMARY,
             DwdObservationResolution.MINUTE_1,

tests/provider/dwd/observation/test_metaindex.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2018-2021, earthobservations developers.
 # Distributed under the MIT License. See LICENSE for more info.
 """ tests for file index creation """
+import aiohttp
 import pytest
 import requests
 
@@ -26,7 +27,9 @@ def test_meta_index_creation():
 
     assert not meta_index.empty
 
-    with pytest.raises(requests.exceptions.HTTPError):
+    with pytest.raises(
+        (requests.exceptions.HTTPError, aiohttp.client_exceptions.ClientResponseError)
+    ):
         create_meta_index_for_climate_observations(
             DwdObservationDataset.CLIMATE_SUMMARY,
             Resolution.MINUTE_1,

tests/provider/dwd/test_index.py

@@ -14,7 +14,7 @@
     DwdObservationPeriod,
     DwdObservationResolution,
 )
-from wetterdienst.util.network import list_remote_files_legacy
+from wetterdienst.util.network import list_remote_files_fsspec
 
 
 def test_build_index_path():
@@ -28,7 +28,7 @@ def test_build_index_path():
 
 @pytest.mark.remote
 def test_list_files_of_climate_observations():
-    files_server = list_remote_files_legacy(
+    files_server = list_remote_files_fsspec(
         "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/"
         "annual/kl/recent",
         recursive=False,

wetterdienst/provider/dwd/forecast/api.py

@@ -41,7 +41,7 @@
 from wetterdienst.util.cache import metaindex_cache
 from wetterdienst.util.enumeration import parse_enumeration_from_template
 from wetterdienst.util.geo import convert_dm_to_dd
-from wetterdienst.util.network import list_remote_files_legacy
+from wetterdienst.util.network import list_remote_files_fsspec
 
 log = logging.getLogger(__name__)
 
@@ -243,7 +243,7 @@ def get_url_for_date(url: str, date: Union[datetime, DwdForecastDate]) -> str:
         Returns:
             file url based on the filtering
         """
-        urls = list_remote_files_legacy(url, False)
+        urls = list_remote_files_fsspec(url, recursive=False)
 
         if date == DwdForecastDate.LATEST:
             try:

wetterdienst/provider/dwd/index.py

@@ -15,11 +15,8 @@
     DWDCDCBase,
 )
 from wetterdienst.provider.dwd.observation.metadata.dataset import DwdObservationDataset
-from wetterdienst.util.cache import (
-    fileindex_cache_five_minutes,
-    fileindex_cache_one_hour,
-)
-from wetterdienst.util.network import list_remote_files_legacy
+from wetterdienst.util.cache import fileindex_cache_five_minutes
+from wetterdienst.util.network import list_remote_files_fsspec
 
 
 def _create_file_index_for_dwd_server(
@@ -43,7 +40,7 @@ def _create_file_index_for_dwd_server(
 
     url = reduce(urljoin, [DWD_SERVER, DWD_CDC_PATH, cdc_base.value, parameter_path])
 
-    files_server = list_remote_files_legacy(url, recursive=True)
+    files_server = list_remote_files_fsspec(url, recursive=True)
 
     files_server = pd.DataFrame(
         files_server, columns=[DwdColumns.FILENAME.value], dtype="str"

wetterdienst/provider/dwd/observation/metaindex.py

@@ -32,7 +32,7 @@
 )
 from wetterdienst.provider.dwd.observation.metadata.dataset import DwdObservationDataset
 from wetterdienst.util.cache import CacheExpiry, metaindex_cache
-from wetterdienst.util.network import download_file, list_remote_files_legacy
+from wetterdienst.util.network import download_file, list_remote_files_fsspec
 
 METADATA_COLUMNS = [
     Columns.STATION_ID.value,
@@ -143,7 +143,7 @@ def _create_meta_index_for_climate_observations(
         ],
     )
 
-    files_server = list_remote_files_legacy(url, recursive=True)
+    files_server = list_remote_files_fsspec(url, recursive=True)
 
     # Find the one meta file from the files listed on the server
     meta_file = _find_meta_file(files_server, url)
@@ -220,7 +220,7 @@ def _create_meta_index_for_1minute_historical_precipitation() -> pd.DataFrame:
         ],
     )
 
-    metadata_file_paths = list_remote_files_legacy(url, recursive=False)
+    metadata_file_paths = list_remote_files_fsspec(url, recursive=False)
 
     station_ids = [
         re.findall(STATION_ID_REGEX, file).pop(0) for file in metadata_file_paths

wetterdienst/util/network.py

@@ -134,7 +134,10 @@ def list_remote_files_fsspec(
     filesystem = NetworkFilesystemManager.get(ttl=ttl)
 
     # Recursively list remote directory.
-    remote_urls = filesystem.expand_path(url, recursive=recursive)
+    if not recursive:
+        remote_urls = filesystem.glob(url + "/*")
+    else:
+        remote_urls = filesystem.expand_path(url, recursive=recursive)
 
     # Only list files, so remove all directories.
     try: