Skip to content

Commit

Permalink
Use logging instead of printing
Browse files Browse the repository at this point in the history
This avoids poisoning STDOUT with log messages which really
should go to STDERR. In turn, this makes room for real output
on STDOUT originating from the upcoming CLI module.
  • Loading branch information
amotl committed Jun 16, 2020
1 parent 27e0fcb commit baec359
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 10 deletions.
7 changes: 5 additions & 2 deletions python_dwd/data_collection.py
@@ -1,4 +1,5 @@
""" Data collection pipeline """
import logging
from pathlib import Path
from typing import List, Union
import pandas as pd
Expand All @@ -12,6 +13,8 @@
from python_dwd.parsing_data.parse_data_from_files import parse_dwd_data
from python_dwd.data_storing import restore_dwd_data, store_dwd_data, _build_local_store_key

log = logging.getLogger(__name__)


def collect_dwd_data(station_ids: List[int],
parameter: Union[Parameter, str],
Expand Down Expand Up @@ -60,13 +63,13 @@ def collect_dwd_data(station_ids: List[int],

# When successful append data and continue with next iteration
if not station_data.empty:
print(f"Data for {request_string} restored from local.")
log.info(f"Data for {request_string} restored from local.")

data.append(station_data)

continue

print(f"Data for {request_string} will be collected from internet.")
log.info(f"Data for {request_string} will be collected from internet.")

remote_files = create_file_list_for_dwd_server(
[station_id], parameter, time_resolution, period_type, folder, create_new_filelist)
Expand Down
9 changes: 6 additions & 3 deletions python_dwd/dwd_station_request.py
@@ -1,3 +1,4 @@
import logging
from pathlib import Path
from typing import List, Union, Optional, Dict, Generator
import pandas as pd
Expand All @@ -16,6 +17,8 @@
from python_dwd.constants.access_credentials import DWD_FOLDER_MAIN
from python_dwd.enumerations.column_names_enumeration import DWDColumns

log = logging.getLogger(__name__)


class DWDStationRequest:
"""
Expand Down Expand Up @@ -80,9 +83,9 @@ def __init__(self,
if not check_parameters(parameter=self.parameter,
time_resolution=self.time_resolution,
period_type=period_type):
print(f"Combination of: parameter {self.parameter.value}, "
f"time_resolution {self.time_resolution.value}, "
f"period_type {period_type} not available and removed.")
log.info(f"Combination of: parameter {self.parameter.value}, "
f"time_resolution {self.time_resolution.value}, "
f"period_type {period_type} not available and removed.")
self.period_type.remove(period_type)

# Use the clean up of self.period_type to identify if there's any data with those parameters
Expand Down
13 changes: 8 additions & 5 deletions python_dwd/parsing_data/parse_data_from_files.py
@@ -1,4 +1,5 @@
""" function to read data from dwd server """
import logging
from typing import List, Tuple
from io import BytesIO
import pandas as pd
Expand All @@ -7,6 +8,8 @@
from python_dwd.constants.column_name_mapping import GERMAN_TO_ENGLISH_COLUMNS_MAPPING
from python_dwd.constants.metadata import NA_STRING, STATIONDATA_SEP

log = logging.getLogger(__name__)


def parse_dwd_data(filenames_and_files: List[Tuple[str, BytesIO]]) -> pd.DataFrame:
"""
Expand All @@ -27,9 +30,9 @@ def parse_dwd_data(filenames_and_files: List[Tuple[str, BytesIO]]) -> pd.DataFra
try:
data = pd.concat(data).reset_index(drop=True)
except ValueError:
print(f"An error occurred while concatenating the data for files "
f"{[filename for filename, file in filenames_and_files]}. "
f"An empty DataFrame will be returned.")
log.error(f"An error occurred while concatenating the data for files "
f"{[filename for filename, file in filenames_and_files]}. "
f"An empty DataFrame will be returned.")
data = pd.DataFrame()

return data
Expand Down Expand Up @@ -57,10 +60,10 @@ def _parse_dwd_data(filename_and_file: Tuple[str, BytesIO]) -> pd.DataFrame:
dtype="str" # dtypes are mapped manually to ensure expected dtypes
)
except pd.errors.ParserError:
print(f"The file representing {filename} could not be parsed and is skipped.")
log.warning(f"The file representing {filename} could not be parsed and is skipped.")
return pd.DataFrame()
except ValueError:
print(f"The file representing {filename} is None and is skipped.")
log.warning(f"The file representing {filename} is None and is skipped.")
return pd.DataFrame()

data = data.rename(columns=str.upper).rename(GERMAN_TO_ENGLISH_COLUMNS_MAPPING)
Expand Down

0 comments on commit baec359

Please sign in to comment.