Fix tests

THIRD_PARTY_NOTICES

@@ -5907,32 +5907,6 @@ Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
 
 
 
-munch
-2.5.0
-MIT License
-Rotem Yaari
-https://github.com/Infinidat/munch
-Copyright (c) 2010 David Schoonover
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
-
 mypy-extensions
 0.4.3
 MIT License

docs/usage/cli.rst

@@ -53,6 +53,7 @@ Command Line Interface
         --language=<language>                 Output language. [Default: en]
         --version                             Show version information
         --tidy                                Tidy DataFrame
+        --humanize                            Humanize parameters
         --si-units                            Convert to SI units
         --pretty                              Pretty json with indent 4
         --debug                               Enable debug messages

pyproject.toml

@@ -81,7 +81,6 @@ pandas = "^1.2"
 numpy = "^1.19.5"
 scipy = "^1.5.2"
 cachetools = "^4.1.1"
-munch = "^2.5.0"
 dateparser = "^1.0.0"
 beautifulsoup4 = "^4.9.1"
 requests = "^2.24.0"

requirements.txt

@@ -62,7 +62,6 @@ measurement==3.2.0
 mistune==0.8.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 mock==4.0.3; python_version >= "3.6"
 mpmath==1.2.1; python_version >= "3.6"
-munch==2.5.0
 mypy-extensions==0.4.3; python_version >= "3.6"
 nbconvert==5.6.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 nbdime==3.1.0; python_version >= "3.6"

tests/provider/dwd/observation/test_api_data.py

@@ -94,8 +94,8 @@ def test_dwd_observation_data_parameter():
 
     assert request.parameter == [
         (
-            DwdObservationDatasetTree.DAILY.PRECIPITATION_MORE.PRECIPITATION_HEIGHT,  # Noqa: E501, B950
-            DwdObservationDataset.PRECIPITATION_MORE,
+            DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.PRECIPITATION_HEIGHT,  # Noqa: E501, B950
+            DwdObservationDataset.CLIMATE_SUMMARY,
         )
     ]
 

tests/provider/dwd/observation/test_parameters.py

@@ -23,12 +23,12 @@
         DwdObservationDataset.CLIMATE_SUMMARY,
     ),
     (
-        DwdObservationDatasetTree.DAILY.PRECIPITATION_MORE.PRECIPITATION_HEIGHT,
-        DwdObservationDataset.PRECIPITATION_MORE,
+        DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.PRECIPITATION_HEIGHT,
+        DwdObservationDataset.CLIMATE_SUMMARY,
     ),
     (
-        DwdObservationDatasetTree.DAILY.PRECIPITATION_MORE.PRECIPITATION_FORM,
-        DwdObservationDataset.PRECIPITATION_MORE,
+        DwdObservationDatasetTree.DAILY.CLIMATE_SUMMARY.PRECIPITATION_FORM,
+        DwdObservationDataset.CLIMATE_SUMMARY,
     ),
 ]
 

tests/ui/test_cli.py

@@ -156,10 +156,7 @@ def invoke_wetterdienst_stations_geo(provider, kind, setting, fmt="json"):
 
 def invoke_wetterdienst_values_static(provider, kind, setting, station, fmt="json"):
     runner = CliRunner()
-    print(
-        f"values --provider={provider} --kind={kind} "
-        f"{setting} --station={station} --format={fmt}"
-    )
+
     result = runner.invoke(
         cli,
         f"values --provider={provider} --kind={kind} "
@@ -207,6 +204,40 @@ def invoke_wetterdienst_values_geo(provider, kind, setting, fmt="json"):
     return result
 
 
+def test_no_provider():
+    runner = CliRunner()
+
+    result = runner.invoke(cli, "stations --provider=abc --kind=abc")
+
+    assert (
+        "Error: Invalid value for '--provider': invalid choice: abc." in result.output
+    )
+
+
+def test_no_kind():
+    runner = CliRunner()
+
+    result = runner.invoke(cli, "stations --provider=dwd --kind=abc")
+
+    assert "Invalid value for '--kind': invalid choice: abc." in result.output
+
+
+def test_data_range(capsys):
+    runner = CliRunner()
+
+    result = runner.invoke(
+        cli,
+        "values --provider=eccc --kind=observation --parameter=precipitation_height "
+        "--resolution=daily --name=toronto",
+    )
+
+    assert isinstance(result.exception, TypeError)
+    assert (
+        "Combination of provider ECCC and kind OBSERVATION requires start and end date"
+        in str(result.exception)
+    )
+
+
 @pytest.mark.parametrize(
     "provider,kind,setting,station_id,station_name",
     SETTINGS_STATIONS,
@@ -277,8 +308,8 @@ def test_cli_stations_excel(
     provider, kind, setting, station_id, station_name, tmpdir_factory
 ):
 
-    filename = tmpdir_factory.mktemp("data").join("stations.xlsx")
-    # filename = "stations.xlsx"
+    # filename = tmpdir_factory.mktemp("data").join("stations.xlsx")
+    filename = "stations.xlsx"
 
     _ = invoke_wetterdienst_stations_export(
         provider=provider,
@@ -384,8 +415,8 @@ def test_cli_values_excel(
     provider, kind, setting, station_id, station_name, tmpdir_factory
 ):
 
-    filename = tmpdir_factory.mktemp("data").join("values.xlsx")
-    # filename = "values.xlsx"
+    # filename = tmpdir_factory.mktemp("data").join("values.xlsx")
+    filename = "values.xlsx"
 
     _ = invoke_wetterdienst_values_export(
         provider=provider,

wetterdienst/core/scalar/request.py

@@ -19,6 +19,7 @@
 from wetterdienst.core.scalar.result import StationsResult
 from wetterdienst.exceptions import InvalidEnumeration, StartDateEndDateError
 from wetterdienst.metadata.columns import Columns
+from wetterdienst.metadata.datarange import DataRange
 from wetterdienst.metadata.kind import Kind
 from wetterdienst.metadata.period import Period, PeriodType
 from wetterdienst.metadata.provider import Provider
@@ -58,12 +59,6 @@ def _resolution_type(self) -> ResolutionType:
         """ Resolution type, multi, fixed, ..."""
         pass
 
-    # TODO: implement for source with dynamic resolution
-    @staticmethod
-    def _determine_resolution(dates: pd.Series) -> Resolution:
-        """ Function to determine resolution from a pandas Series of dates """
-        pass
-
     @property
     def frequency(self) -> Frequency:
         """Frequency for the given resolution, used to create a full date range for
@@ -89,6 +84,13 @@ def _parameter_base(self) -> Enum:
         DWDObservationParameter"""
         pass
 
+    @property
+    @abstractmethod
+    def _data_range(self) -> DataRange:
+        """State whether data from this provider is given in fixed data chunks
+        or has to be defined over start and end date"""
+        pass
+
     @property
     @abstractmethod
     def _has_datasets(self) -> bool:

wetterdienst/core/scalar/result.py

@@ -28,7 +28,6 @@ def __init__(
     ) -> None:
         # TODO: add more attributes from ScalarStations class
         self.stations = stations
-
         self.df = df
         self._kwargs = kwargs
 

wetterdienst/core/scalar/values.py

@@ -301,6 +301,9 @@ def _create_empty_station_parameter_df(
                 parameter = [*dataset_tree[resolution.name][dataset.name]]
 
         if self.stations.stations.tidy:
+            if not self.stations.stations.start_date:
+                return pd.DataFrame(None, columns=self._meta_fields)
+
             data = []
             for par in pd.Series(parameter):
                 if par.name.startswith("QUALITY"):
@@ -320,11 +323,15 @@ def _create_empty_station_parameter_df(
 
             return df
         else:
-            df = self._base_df
             parameter = pd.Series(parameter).map(lambda x: x.value).tolist()
 
             # Base columns
-            columns = [Columns.STATION_ID.value, Columns.DATE.value, *parameter]
+            columns = [*self._meta_fields, *parameter]
+
+            if self.stations.stations.start_date:
+                return pd.DataFrame(None, columns=columns)
+
+            df = self._base_df
 
             df = df.reindex(columns=columns)
 
@@ -443,11 +450,6 @@ def query(self) -> Generator[ValuesResult, None, None]:
                 parameter_df = self._collect_station_parameter(station_id, parameter)
 
                 if parameter_df.empty:
-                    parameter_df = self._create_empty_station_parameter_df(
-                        station_id, parameter
-                    )
-
-                    station_data.append(parameter_df)
                     continue
 
                 # Merge on full date range if values are found to ensure result
@@ -476,21 +478,21 @@ def query(self) -> Generator[ValuesResult, None, None]:
 
                 station_data.append(parameter_df)
 
-            station_df = pd.concat(station_data, ignore_index=True)
+            try:
+                station_df = pd.concat(station_data, ignore_index=True)
+            except ValueError:
+                station_df = self._create_empty_station_parameter_df(
+                    station_id, parameter
+                )
 
             station_df = self._coerce_meta_fields(station_df)
 
             # Filter for dates range if start_date and end_date are defined
-            if self.stations.start_date:
-                # df_station may be empty depending on if station has data for given
-                # constraints
-                try:
-                    station_df = station_df[
-                        (station_df[Columns.DATE.value] >= self.stations.start_date)
-                        & (station_df[Columns.DATE.value] <= self.stations.end_date)
-                    ]
-                except KeyError:
-                    pass
+            if not station_df.empty and self.stations.start_date:
+                station_df = station_df[
+                    (station_df[Columns.DATE.value] >= self.stations.start_date)
+                    & (station_df[Columns.DATE.value] <= self.stations.end_date)
+                ]
 
             station_df = self._coerce_parameter_types(station_df)
 

wetterdienst/metadata/datarange.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2018-2021, earthobservations developers.
+# Distributed under the MIT License. See LICENSE for more info.
+from enum import Enum
+
+
+class DataRange(Enum):
+    """Enumeration for data range. This is required for querying data
+    which can not be queried in a fixed file format and must be defined
+    over start and end date"""
+
+    FIXED = "fixed"
+    LOOSELY = "loosely"

wetterdienst/provider/dwd/forecast/api.py

@@ -15,7 +15,9 @@
 from wetterdienst.core.scalar.request import ScalarRequestCore
 from wetterdienst.core.scalar.result import StationsResult, ValuesResult
 from wetterdienst.core.scalar.values import ScalarValuesCore
+from wetterdienst.exceptions import InvalidParameter
 from wetterdienst.metadata.columns import Columns
+from wetterdienst.metadata.datarange import DataRange
 from wetterdienst.metadata.kind import Kind
 from wetterdienst.metadata.period import Period, PeriodType
 from wetterdienst.metadata.provider import Provider
@@ -351,7 +353,7 @@ class DwdMosmixRequest(ScalarRequestCore):
     _resolution_base = Resolution  # use general Resolution for fixed Resolution
     _period_type = PeriodType.FIXED
     _period_base = None
-
+    _data_range = DataRange.FIXED
     _has_datasets = True
     _unique_dataset = True
     _dataset_base = DwdMosmixDataset
@@ -452,6 +454,14 @@ def __init__(
             si_units=si_units,
         )
 
+        if not start_issue:
+            start_issue = DwdForecastDate.LATEST
+
+        try:
+            start_issue = parse_enumeration_from_template(start_issue, DwdForecastDate)
+        except InvalidParameter:
+            pass
+
         # Parse issue date if not set to fixed "latest" string
         if start_issue is DwdForecastDate.LATEST and end_issue:
             log.info(