Fix bug in unit conversion

Add tests for unit onversion

tests/provider/dwd/observation/test_api_data.py

@@ -381,16 +381,84 @@ def test_dwd_observation_data_result_tabular():
 
 
 @pytest.mark.remote
-def test_dwd_observation_data_result_tidy():
-    """ Test for actual values (tidy) """
+def test_dwd_observation_data_result_tabular_metric():
+    """ Test for actual values (tabular) in metric units """
+    request = DwdObservationRequest(
+        parameter=[DwdObservationDataset.CLIMATE_SUMMARY],
+        resolution=DwdObservationResolution.DAILY,
+        start_date="1933-12-31",  # few days before official start
+        end_date="1934-01-01",  # few days after official start,
+        tidy=False,
+        humanize=False,
+        metric=True,
+    ).filter_by_station_id(
+        station_id=[1048],
+    )
+
+    df = request.values.all().df
+
+    assert list(df.columns.values) == [
+        "date",
+        "station_id",
+        "qn_3",
+        "fx",
+        "fm",
+        "qn_4",
+        "rsk",
+        "rskf",
+        "sdk",
+        "shk_tag",
+        "nm",
+        "vpm",
+        "pm",
+        "tmk",
+        "upm",
+        "txk",
+        "tnk",
+        "tgk",
+    ]
+
+    assert_frame_equal(
+        df,
+        pd.DataFrame(
+            {
+                "date": [
+                    datetime(1933, 12, 31, tzinfo=pytz.UTC),
+                    datetime(1934, 1, 1, tzinfo=pytz.UTC),
+                ],
+                "station_id": pd.Categorical(["01048", "01048"]),
+                "qn_3": pd.Series([pd.NA, pd.NA], dtype=pd.Int64Dtype()),
+                "fx": pd.to_numeric([pd.NA, pd.NA], errors="coerce"),
+                "fm": pd.to_numeric([pd.NA, pd.NA], errors="coerce"),
+                "qn_4": pd.Series([pd.NA, 1], dtype=pd.Int64Dtype()),
+                "rsk": pd.to_numeric([pd.NA, 0.2], errors="coerce"),
+                "rskf": pd.to_numeric([pd.NA, 8], errors="coerce"),
+                "sdk": pd.to_numeric([pd.NA, pd.NA], errors="coerce"),
+                "shk_tag": pd.Series([pd.NA, 0], dtype=pd.Int64Dtype()),
+                "nm": pd.to_numeric([pd.NA, 100.0], errors="coerce"),
+                "vpm": pd.to_numeric([pd.NA, 640.0], errors="coerce"),
+                "pm": pd.to_numeric([pd.NA, 100860.0], errors="coerce"),
+                "tmk": pd.to_numeric([pd.NA, 273.65], errors="coerce"),
+                "upm": pd.to_numeric([pd.NA, 97.00], errors="coerce"),
+                "txk": pd.to_numeric([pd.NA, 273.84999999999997], errors="coerce"),
+                "tnk": pd.to_numeric([pd.NA, 273.34999999999997], errors="coerce"),
+                "tgk": pd.to_numeric([pd.NA, pd.NA], errors="coerce"),
+            }
+        ),
+    )
+
+
+@pytest.mark.remote
+def test_dwd_observation_data_result_tidy_metric():
+    """ Test for actual values (tidy) in metric units """
     request = DwdObservationRequest(
         parameter=[DwdObservationDataset.CLIMATE_SUMMARY],
         resolution=DwdObservationResolution.DAILY,
         start_date="1933-12-31",  # few days before official start
         end_date="1934-01-01",  # few days after official start,
         tidy=True,
         humanize=False,
-        metric=False,
+        metric=True,
     ).filter_by_station_id(
         station_id=(1048,),
     )
@@ -558,25 +626,25 @@ def test_dwd_observation_data_result_tidy():
                         0,
                         # NM
                         pd.NA,
-                        8.0,
+                        100.0,
                         # VPM
                         pd.NA,
-                        6.4,
+                        640.0,
                         # PM
                         pd.NA,
-                        1008.60,
+                        100860.0,
                         # TMK
                         pd.NA,
-                        0.5,
+                        273.65,
                         # UPM
                         pd.NA,
                         97.00,
                         # TXK
                         pd.NA,
-                        0.7,
+                        273.84999999999997,
                         # TNK
                         pd.NA,
-                        0.2,
+                        273.34999999999997,
                         # TGK
                         pd.NA,
                         pd.NA,

tests/provider/dwd/radar/test_api_most_recent.py

@@ -102,7 +102,7 @@ def test_radar_request_site_most_recent_sweep_vol_v_hdf5():
     assert hdf["/how"].attrs.get("scan_count") == 10
     assert hdf["/dataset1/how"].attrs.get("scan_index") == 1
 
-    assert hdf["/dataset1/data1/data"].shape == (360, 720)
+    assert hdf["/dataset1/data1/data"].shape in ((360, 720), (361, 720))
 
     # Verify that the second file is the second scan / elevation level.
     buffer = results[1].data

wetterdienst/core/scalar/request.py

@@ -482,9 +482,14 @@ def discover(cls, filter_=None, dataset=None, flatten: bool = True) -> str:
         return json.dumps(parameters, indent=4, ensure_ascii=False)
 
     @classmethod
-    def _setup_discover_filter(cls, filter_):
-        """Helper method to create filter for discover method, can be overwritten by
-        subclasses to use other then the resolution for filtering"""
+    def _setup_discover_filter(cls, filter_) -> list:
+        """
+            Helper method to create filter for discover method, can be overwritten by
+            subclasses to use other then the resolution for filtering
+
+        :param filter_: typically resolution, if used in subclass can be directed towards something else
+        :return:
+        """
         if cls._resolution_type == ResolutionType.FIXED:
             log.warning("resolution filter will be ignored due to fixed resolution")
 

wetterdienst/core/scalar/values.py

@@ -101,6 +101,8 @@ def _determine_resolution(series: pd.Series) -> Resolution:
         pass
 
     def convert_values_to_metric(self, df: pd.DataFrame, dataset) -> pd.DataFrame:
+        """ Function to convert values to metric units with help of conversion factors """
+
         def _convert_values_to_metric(series):
             op, factor = conversion_factors.get(series.name, (None, None))
 
@@ -112,9 +114,9 @@ def _convert_values_to_metric(series):
         conversion_factors = self._create_conversion_factors(dataset)
 
         if self.stations.stations.tidy:
-            df.loc[:, Columns.VALUE.value] = df.groupby(Columns.PARAMETER.value)[
-                Columns.VALUE.value
-            ].apply(_convert_values_to_metric)
+            df.loc[:, Columns.VALUE.value] = df.groupby(
+                Columns.PARAMETER.value, sort=False
+            )[Columns.VALUE.value].apply(_convert_values_to_metric)
         else:
             df = df.apply(_convert_values_to_metric, axis=0)
 
@@ -504,7 +506,7 @@ def _coerce_dates(self, series: pd.Series) -> pd.Series:
     def _coerce_integers(series: pd.Series) -> pd.Series:
         """Method to parse integers for type coercion. Uses pandas.Int64Dtype() to
         allow missing values."""
-        return pd.to_numeric(series, errors="coerce").astype(pd.Int64Dtype())
+        return pd.to_numeric(series, errors="coerce").astype(pd.Float64Dtype()).astype(pd.Int64Dtype())
 
     @staticmethod
     def _coerce_strings(series: pd.Series) -> pd.Series:

wetterdienst/provider/dwd/observation/fileindex.py

@@ -82,7 +82,7 @@ def create_file_index_for_climate_observations(
 
     file_index = file_index.dropna().reset_index(drop=True)
 
-    file_index[DwdColumns.STATION_ID.value] = file_index[
+    file_index.loc[:, DwdColumns.STATION_ID.value] = file_index[
         DwdColumns.STATION_ID.value
     ].astype(str)