Merge pull request earthobservations#66 from earthobservations/bug_fi…

…x_series_to_list Bug fix series to list
panodata · Jun 20, 2020 · 90e96b7 · 90e96b7
2 parents 575c867 + 833b7f8
commit 90e96b7
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 8 deletions.
diff --git a/python_dwd/constants/access_credentials.py b/python_dwd/constants/access_credentials.py
@@ -4,6 +4,7 @@
 DWD_USER = 'anonymous'
 DWD_PASSWORD = ''
 FTP_EXPRESSION = 'ftp://'
+HTTPS_EXPRESSION = 'https://'
 
 DWD_FOLDER_MAIN = './dwd_data'
 DWD_FOLDER_METADATA = 'metadata'

diff --git a/python_dwd/download/download.py b/python_dwd/download/download.py
@@ -12,13 +12,14 @@
 from python_dwd.download.download_services import create_remote_file_name
 from python_dwd.additionals.functions import find_all_matchstrings_in_string
 from python_dwd.enumerations.column_names_enumeration import DWDMetaColumns
+from python_dwd.exceptions.failed_download_exception import FailedDownload
 
 
 def download_dwd_data(remote_files: pd.DataFrame,
                       parallel_download: bool = False) -> List[Tuple[str, BytesIO]]:
     """ wrapper for _download_dwd_data to provide a multiprocessing feature"""
 
-    remote_files: List[str] = remote_files[DWDMetaColumns.FILENAME.value].to_list()
+    remote_files = remote_files[DWDMetaColumns.FILENAME.value].values.tolist()
 
     if parallel_download:
         return list(
@@ -52,10 +53,10 @@ def _download_dwd_data(remote_file: Union[str, Path]) -> BytesIO:
     try:
         with urllib.request.urlopen(file_server) as url_request:
             zip_file = BytesIO(url_request.read())
-    except urllib.error.URLError:
-        raise urllib.error.URLError(f"Error: the stationdata {file_server} couldn't be reached.")
-    except urllib.error.HTTPError:
-        pass
+    except urllib.error.URLError as e:
+        raise e(f"Error: the stationdata {file_server} couldn't be reached.")
+    except:
+        raise FailedDownload(f"Download failed for {file_server}")
 
     try:
         with zipfile.ZipFile(zip_file) as zip_file_opened:

diff --git a/python_dwd/download/download_services.py b/python_dwd/download/download_services.py
@@ -2,7 +2,7 @@
 from pathlib import Path, PurePosixPath
 from typing import Union
 
-from python_dwd.constants.access_credentials import DWD_FOLDER_STATIONDATA, DWD_SERVER, DWD_PATH, FTP_EXPRESSION
+from python_dwd.constants.access_credentials import DWD_FOLDER_STATIONDATA, DWD_SERVER, DWD_PATH, HTTPS_EXPRESSION
 
 
 def create_local_file_name(remote_file_path: Union[Path, str],
@@ -32,4 +32,4 @@ def create_remote_file_name(file: str) -> str:
                                 DWD_PATH,
                                 file)
 
-    return f"{FTP_EXPRESSION}{file_server}"
+    return f"{HTTPS_EXPRESSION}{file_server}"
diff --git a/python_dwd/enumerations/column_names_enumeration.py b/python_dwd/enumerations/column_names_enumeration.py
@@ -42,6 +42,10 @@ class DWDOrigColumns(Enum):
     TNK = "TNK"
     TGK = "TGK"
 
+    # 10 minutes data
+    FX_10 = "FX_10"
+    DX_10 = "DX_10"
+
 
 class DWDMetaColumns(Enum):
     """ Overhauled column names for metadata fields """
@@ -63,7 +67,7 @@ class DWDMetaColumns(Enum):
 
 class DWDDataColumns(Enum):
     """ Overhauled column names for data fields """
-
+    DATE = "DATE"
     # Daily climate summary
     FX = "WIND_GUST_MAX"
     FM = "WIND_VELOCITY"

diff --git a/python_dwd/exceptions/failed_download_exception.py b/python_dwd/exceptions/failed_download_exception.py
@@ -0,0 +1,6 @@
+""" Exception that can be thrown if download fails"""
+
+
+class FailedDownload(Exception):
+    """ Failed download exception """
+    pass