Use meaningful column names, starting with daily climate summary data

From "column_name_mapping.py" and "column_names_enumeration.py",
we are seeing the intention to map meteorological short identifiers
to more meaningful English long names.

This complements the current implementation by providing appropriate
mappings for daily climate summary data (kl) and also adds an
appropriate test case for that.

While being on it, we discovered that "_parse_dwd_data" as well as
"collect_dwd_data" somehow wouldn't actually account for column names
to be propagated, so we adjusted some spots on data frame handling.

Trivia:
- The test case has been marked as "remote" to be able to tell unit
  tests based on fixtures and full integration tests apart.
- When massaging the data frame after parsing data from CSV,
  the "EOR" column gets dropped right away as it actually
  has no real value on downstream processing.

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+
+markers =
+    remote: Tests accessing the internet.

python_dwd/constants/column_name_mapping.py

@@ -13,7 +13,23 @@
     DWDOrigColumns.LONGITUDE.value:                 DWDColumns.LONGITUDE.value,
     DWDOrigColumns.LONGITUDE_ALTERNATIVE.value:     DWDColumns.LONGITUDE.value,
     DWDOrigColumns.STATIONNAME.value:               DWDColumns.STATIONNAME.value,
-    DWDOrigColumns.STATE.value:                     DWDColumns.STATE.value
+    DWDOrigColumns.STATE.value:                     DWDColumns.STATE.value,
+
+    # Daily climate summary
+    DWDOrigColumns.FX.value:                        DWDColumns.FX.value,
+    DWDOrigColumns.FM.value:                        DWDColumns.FM.value,
+    DWDOrigColumns.RSK.value:                       DWDColumns.RSK.value,
+    DWDOrigColumns.RSKF.value:                      DWDColumns.RSKF.value,
+    DWDOrigColumns.SDK.value:                       DWDColumns.SDK.value,
+    DWDOrigColumns.SHK_TAG.value:                   DWDColumns.SHK_TAG.value,
+    DWDOrigColumns.NM.value:                        DWDColumns.NM.value,
+    DWDOrigColumns.VPM.value:                       DWDColumns.VPM.value,
+    DWDOrigColumns.PM.value:                        DWDColumns.PM.value,
+    DWDOrigColumns.TMK.value:                       DWDColumns.TMK.value,
+    DWDOrigColumns.UPM.value:                       DWDColumns.UPM.value,
+    DWDOrigColumns.TXK.value:                       DWDColumns.TXK.value,
+    DWDOrigColumns.TNK.value:                       DWDColumns.TNK.value,
+    DWDOrigColumns.TGK.value:                       DWDColumns.TGK.value,
 }
 
 METADATA_DTYPE_MAPPING = {

python_dwd/data_collection.py

@@ -84,4 +84,4 @@ def collect_dwd_data(station_ids: List[int],
 
         data.append(station_data)
 
-    return pd.concat(data, axis=1, ignore_index=True)
+    return pd.concat(data)

python_dwd/enumerations/column_names_enumeration.py

@@ -24,6 +24,22 @@ class DWDOrigColumns(Enum):
     STATIONNAME = "STATIONSNAME"
     STATE = "BUNDESLAND"
 
+    # Daily climate summary
+    FX = "FX"
+    FM = "FM"
+    RSK = "RSK"
+    RSKF = "RSKF"
+    SDK = "SDK"
+    SHK_TAG = "SHK_TAG"
+    NM = "NM"
+    VPM = "VPM"
+    PM = "PM"
+    TMK = "TMK"
+    UPM = "UPM"
+    TXK = "TXK"
+    TNK = "TNK"
+    TGK = "TGK"
+
 
 class DWDColumns(Enum):
     """ Overhauled column names for the library """
@@ -41,3 +57,19 @@ class DWDColumns(Enum):
     FILENAME = "FILENAME"
     HAS_FILE = "HAS_FILE"
     FILEID = "FILEID"
+
+    # Daily climate summary
+    FX = "WIND_GUST_MAX"
+    FM = "WIND_VELOCITY"
+    RSK = "PRECIPITATION_HEIGHT"
+    RSKF = "PRECIPITATION_FORM"
+    SDK = "SUNSHINE_DURATION"
+    SHK_TAG = "SNOW_DEPTH"
+    NM = "CLOUD_COVER"
+    VPM = "VAPOR_PRESSURE"
+    PM = "PRESSURE"
+    TMK = "TEMPERATURE"
+    UPM = "HUMIDITY"
+    TXK = "TEMPERATURE_MAX_200"
+    TNK = "TEMPERATURE_MIN_200"
+    TGK = "TEMPERATURE_MIN_005"

python_dwd/parsing_data/parse_data_from_files.py

@@ -66,6 +66,19 @@ def _parse_dwd_data(filename_and_file: Tuple[str, BytesIO]) -> pd.DataFrame:
         log.warning(f"The file representing {filename} is None and is skipped.")
         return pd.DataFrame()
 
-    data = data.rename(columns=str.upper).rename(GERMAN_TO_ENGLISH_COLUMNS_MAPPING)
+    # Column names contain spaces, so strip them away.
+    data = data.rename(columns=str.strip)
 
-    return data.astype(create_stationdata_dtype_mapping(data.columns))
+    # Make column names uppercase.
+    data = data.rename(columns=str.upper)
+
+    # End of record (EOR) has no value, so drop it right away.
+    data = data.drop(columns='EOR', errors='ignore')
+
+    # Assign meaningful column names.
+    data = data.rename(columns=GERMAN_TO_ENGLISH_COLUMNS_MAPPING)
+
+    # Coerce the data types appropriately.
+    data = data.astype(create_stationdata_dtype_mapping(data.columns))
+
+    return data

tests/test_data_collection.py

@@ -1,5 +1,6 @@
 """ Tests for data_collection """
 import mock
+import pytest
 from mock import patch
 from pathlib import Path
 import pandas as pd
@@ -115,3 +116,36 @@ def test_collect_dwd_data_empty():
         write_file=False,
         create_new_filelist=create_new_filelist
     ).empty
+
+
+@pytest.mark.remote
+def test_fetch_and_parse_dwd_data():
+    """ Test for data collection with real data """
+
+    data = collect_dwd_data(
+        station_ids=[1048],
+        parameter=Parameter.CLIMATE_SUMMARY,
+        time_resolution=TimeResolution.DAILY,
+        period_type=PeriodType.RECENT,
+    )
+
+    assert list(data.columns.values) == [
+        'STATION_ID',
+        'DATE',
+        'QN_3',
+        'WIND_GUST_MAX',
+        'WIND_VELOCITY',
+        'QN_4',
+        'PRECIPITATION_HEIGHT',
+        'PRECIPITATION_FORM',
+        'SUNSHINE_DURATION',
+        'SNOW_DEPTH',
+        'CLOUD_COVER',
+        'VAPOR_PRESSURE',
+        'PRESSURE',
+        'TEMPERATURE',
+        'HUMIDITY',
+        'TEMPERATURE_MAX_200',
+        'TEMPERATURE_MIN_200',
+        'TEMPERATURE_MIN_005',
+    ]